From dd58ef019b700900793a1eb48b52123db01b654e Mon Sep 17 00:00:00 2001 From: Dimitry Andric Date: Wed, 30 Dec 2015 11:46:15 +0000 Subject: Vendor import of llvm trunk r256633: https://llvm.org/svn/llvm-project/llvm/trunk@256633 --- test/CodeGen/X86/2006-10-02-BoolRetCrash.ll | 1 + .../X86/2006-10-19-SwitchUnnecessaryBranching.ll | 4 +- test/CodeGen/X86/2007-09-06-ExtWeakAliasee.ll | 2 +- test/CodeGen/X86/2008-03-12-ThreadLocalAlias.ll | 2 +- test/CodeGen/X86/2008-03-14-SpillerCrash.ll | 2 +- .../CodeGen/X86/2008-06-13-NotVolatileLoadStore.ll | 8 +- test/CodeGen/X86/2008-09-05-sinttofp-2xi32.ll | 15 +- test/CodeGen/X86/2009-02-12-DebugInfoVLA.ll | 8 +- test/CodeGen/X86/2009-05-23-dagcombine-shifts.ll | 14 +- test/CodeGen/X86/2009-06-03-Win64SpillXMM.ll | 8 +- test/CodeGen/X86/2009-06-05-VariableIndexInsert.ll | 1 + test/CodeGen/X86/2009-06-06-ConcatVectors.ll | 1 + test/CodeGen/X86/2009-10-16-Scope.ll | 6 +- test/CodeGen/X86/2010-01-18-DbgValue.ll | 8 +- test/CodeGen/X86/2010-02-01-DbgValueCrash.ll | 8 +- test/CodeGen/X86/2010-05-25-DotDebugLoc.ll | 22 +- test/CodeGen/X86/2010-05-26-DotDebugLoc.ll | 20 +- test/CodeGen/X86/2010-05-28-Crash.ll | 18 +- test/CodeGen/X86/2010-06-01-DeadArg-DbgInfo.ll | 28 +- test/CodeGen/X86/2010-07-06-DbgCrash.ll | 7 +- test/CodeGen/X86/2010-08-04-StackVariable.ll | 24 +- test/CodeGen/X86/2010-09-16-EmptyFilename.ll | 10 +- test/CodeGen/X86/2010-11-02-DbgParameter.ll | 8 +- test/CodeGen/X86/2011-01-24-DbgValue-Before-Use.ll | 20 +- test/CodeGen/X86/2011-10-21-widen-cmp.ll | 42 +- .../X86/2011-12-06-AVXVectorExtractCombine.ll | 13 +- test/CodeGen/X86/2011-20-21-zext-ui2fp.ll | 14 +- test/CodeGen/X86/2012-01-12-extract-sv.ll | 28 +- test/CodeGen/X86/2012-08-17-legalizer-crash.ll | 3 +- test/CodeGen/X86/2012-1-10-buildvector.ll | 1 + test/CodeGen/X86/2012-11-30-handlemove-dbg.ll | 8 +- test/CodeGen/X86/2012-11-30-misched-dbg.ll | 16 +- test/CodeGen/X86/2012-11-30-regpres-dbg.ll | 8 +- test/CodeGen/X86/3dnow-intrinsics.ll | 4 +- test/CodeGen/X86/GC/alloc_loop.ll | 1 + test/CodeGen/X86/GC/cg-O0.ll | 1 + test/CodeGen/X86/GC/dynamic-frame-size.ll | 10 +- test/CodeGen/X86/GC/lower_gcroot.ll | 1 + test/CodeGen/X86/MachineBranchProb.ll | 4 +- test/CodeGen/X86/MachineSink-DbgValue.ll | 12 +- test/CodeGen/X86/MergeConsecutiveStores.ll | 37 +- test/CodeGen/X86/StackColoring-dbg.ll | 6 +- test/CodeGen/X86/add-nsw-sext.ll | 168 + test/CodeGen/X86/aliases.ll | 26 +- test/CodeGen/X86/and-encoding.ll | 41 + test/CodeGen/X86/atomic-flags.ll | 61 + test/CodeGen/X86/atomic-minmax-i6432.ll | 8 +- test/CodeGen/X86/atomic-non-integer.ll | 108 + test/CodeGen/X86/atomic128.ll | 52 +- test/CodeGen/X86/atomic_mi.ll | 662 +- test/CodeGen/X86/avg.ll | 724 + test/CodeGen/X86/avx-cvt-2.ll | 1 + test/CodeGen/X86/avx-cvt.ll | 6 +- test/CodeGen/X86/avx-intrinsics-x86-upgrade.ll | 66 + test/CodeGen/X86/avx-intrinsics-x86.ll | 685 +- test/CodeGen/X86/avx-isa-check.ll | 570 + test/CodeGen/X86/avx-load-store.ll | 4 +- test/CodeGen/X86/avx-logic.ll | 2 + test/CodeGen/X86/avx-shift.ll | 1 + test/CodeGen/X86/avx-shuffle-x86_32.ll | 26 +- test/CodeGen/X86/avx-splat.ll | 114 +- test/CodeGen/X86/avx-vbroadcast.ll | 261 +- test/CodeGen/X86/avx-vperm2x128.ll | 44 +- test/CodeGen/X86/avx-win64.ll | 2 - test/CodeGen/X86/avx.ll | 6 +- test/CodeGen/X86/avx2-conversions.ll | 131 +- test/CodeGen/X86/avx2-intrinsics-x86-upgrade.ll | 120 + test/CodeGen/X86/avx2-intrinsics-x86.ll | 94 +- test/CodeGen/X86/avx2-nontemporal.ll | 17 +- test/CodeGen/X86/avx2-vbroadcast.ll | 441 +- test/CodeGen/X86/avx512-arith.ll | 358 +- test/CodeGen/X86/avx512-bugfix-25270.ll | 35 + test/CodeGen/X86/avx512-build-vector.ll | 1 + test/CodeGen/X86/avx512-calling-conv.ll | 481 +- test/CodeGen/X86/avx512-cvt.ll | 119 +- test/CodeGen/X86/avx512-ext.ll | 1835 +++ test/CodeGen/X86/avx512-extract-subvector.ll | 56 + test/CodeGen/X86/avx512-fma.ll | 155 +- test/CodeGen/X86/avx512-gather-scatter-intrin.ll | 185 +- test/CodeGen/X86/avx512-insert-extract.ll | 519 +- test/CodeGen/X86/avx512-intrinsics.ll | 4965 +++++- test/CodeGen/X86/avx512-logic.ll | 164 +- test/CodeGen/X86/avx512-mask-op.ll | 1472 +- test/CodeGen/X86/avx512-skx-insert-subvec.ll | 135 + test/CodeGen/X86/avx512-trunc-ext.ll | 961 -- test/CodeGen/X86/avx512-trunc.ll | 488 + test/CodeGen/X86/avx512-vbroadcast.ll | 262 +- test/CodeGen/X86/avx512-vec-cmp.ll | 27 +- test/CodeGen/X86/avx512bw-intrinsics.ll | 2674 ++- test/CodeGen/X86/avx512bwvl-intrinsics.ll | 748 + test/CodeGen/X86/avx512cd-intrinsics.ll | 18 + test/CodeGen/X86/avx512cdvl-intrinsics.ll | 179 + test/CodeGen/X86/avx512dq-intrinsics.ll | 667 + test/CodeGen/X86/avx512dqvl-intrinsics.ll | 818 +- test/CodeGen/X86/avx512vl-intrinsics.ll | 2977 +++- test/CodeGen/X86/bit-piece-comment.ll | 64 + test/CodeGen/X86/bitreverse.ll | 22 + test/CodeGen/X86/branchfolding-catchpads.ll | 95 + test/CodeGen/X86/buildvec-insertvec.ll | 1 + test/CodeGen/X86/catchpad-realign-savexmm.ll | 53 + test/CodeGen/X86/catchpad-regmask.ll | 144 + test/CodeGen/X86/catchpad-weight.ll | 82 + test/CodeGen/X86/catchret-empty-fallthrough.ll | 53 + test/CodeGen/X86/catchret-fallthrough.ll | 42 + test/CodeGen/X86/cleanuppad-inalloca.ll | 68 + test/CodeGen/X86/cleanuppad-large-codemodel.ll | 27 + test/CodeGen/X86/cleanuppad-realign.ll | 78 + test/CodeGen/X86/clz.ll | 148 +- test/CodeGen/X86/cmp.ll | 44 + test/CodeGen/X86/cmpxchg-clobber-flags.ll | 150 +- test/CodeGen/X86/coal-sections.ll | 23 + test/CodeGen/X86/coalescer-win64.ll | 16 + .../CodeGen/X86/code_placement_cold_loop_blocks.ll | 122 + .../code_placement_ignore_succ_in_inner_loop.ll | 123 + test/CodeGen/X86/code_placement_loop_rotation.ll | 80 + test/CodeGen/X86/code_placement_loop_rotation2.ll | 122 + test/CodeGen/X86/codegen-prepare-cast.ll | 2 +- test/CodeGen/X86/coff-comdat.ll | 2 +- test/CodeGen/X86/combine-and.ll | 1 + test/CodeGen/X86/combine-avx-intrinsics.ll | 59 - test/CodeGen/X86/combine-avx2-intrinsics.ll | 74 - test/CodeGen/X86/combine-multiplies.ll | 163 + test/CodeGen/X86/combine-or.ll | 1 + test/CodeGen/X86/combine-sse2-intrinsics.ll | 53 - test/CodeGen/X86/combine-sse41-intrinsics.ll | 91 - test/CodeGen/X86/commute-two-addr.ll | 2 +- test/CodeGen/X86/constant-hoisting-and.ll | 19 + test/CodeGen/X86/constant-hoisting-cmp.ll | 25 + test/CodeGen/X86/copysign-constant-magnitude.ll | 24 +- test/CodeGen/X86/cppeh-nounwind.ll | 35 - test/CodeGen/X86/cxx_tlscc64.ll | 71 + test/CodeGen/X86/dag-fmf-cse.ll | 22 + test/CodeGen/X86/dag-merge-fast-accesses.ll | 90 + test/CodeGen/X86/darwin-tls.ll | 28 + .../X86/dbg-changes-codegen-branch-folding.ll | 48 +- test/CodeGen/X86/dbg-changes-codegen.ll | 9 +- test/CodeGen/X86/dbg-combine.ll | 12 +- test/CodeGen/X86/debugloc-argsize.ll | 58 + test/CodeGen/X86/divide-by-constant.ll | 32 + test/CodeGen/X86/dllexport-x86_64.ll | 10 +- test/CodeGen/X86/dllexport.ll | 8 +- test/CodeGen/X86/dwarf-comp-dir.ll | 2 +- test/CodeGen/X86/dynamic-allocas-VLAs.ll | 2 +- test/CodeGen/X86/eh-null-personality.ll | 25 + test/CodeGen/X86/eh_frame.ll | 4 +- test/CodeGen/X86/emutls-pic.ll | 168 + test/CodeGen/X86/emutls-pie.ll | 131 + test/CodeGen/X86/emutls.ll | 347 + test/CodeGen/X86/emutls_generic.ll | 107 + test/CodeGen/X86/exedeps-movq.ll | 19 + test/CodeGen/X86/expand-vr64-gr64-copy.mir | 36 + .../X86/extractelement-legalization-cycle.ll | 21 + test/CodeGen/X86/extractelement-shuffle.ll | 1 + test/CodeGen/X86/fadd-combines.ll | 224 + test/CodeGen/X86/fast-isel-bitcasts-avx.ll | 244 + test/CodeGen/X86/fast-isel-bitcasts.ll | 245 + test/CodeGen/X86/fast-isel-cmp-branch.ll | 17 +- test/CodeGen/X86/fast-isel-deadcode.ll | 147 + test/CodeGen/X86/fast-isel-emutls.ll | 48 + test/CodeGen/X86/fast-isel-nontemporal.ll | 111 + test/CodeGen/X86/fast-isel-stackcheck.ll | 44 + test/CodeGen/X86/fast-isel-tls.ll | 2 +- test/CodeGen/X86/fdiv-combine.ll | 69 +- test/CodeGen/X86/fdiv.ll | 52 +- test/CodeGen/X86/fixup-lea.ll | 34 + test/CodeGen/X86/float-asmprint.ll | 15 + test/CodeGen/X86/floor-soft-float.ll | 2 +- test/CodeGen/X86/fma-commute-x86.ll | 761 + test/CodeGen/X86/fma-do-not-commute.ll | 2 +- test/CodeGen/X86/fma-intrinsics-phi-213-to-231.ll | 499 +- test/CodeGen/X86/fma-intrinsics-x86.ll | 688 +- test/CodeGen/X86/fma-scalar-memfold.ll | 383 + test/CodeGen/X86/fma_patterns.ll | 1301 +- test/CodeGen/X86/fma_patterns_wide.ll | 851 +- test/CodeGen/X86/fmaxnum.ll | 203 +- test/CodeGen/X86/fminnum.ll | 181 +- test/CodeGen/X86/fmul-combines.ll | 44 +- test/CodeGen/X86/fold-load-binops.ll | 1 + test/CodeGen/X86/fold-load-unops.ll | 1 + test/CodeGen/X86/fold-push.ll | 40 + test/CodeGen/X86/force-align-stack-alloca.ll | 2 +- test/CodeGen/X86/force-align-stack.ll | 2 +- test/CodeGen/X86/fp-fast.ll | 1 + test/CodeGen/X86/fp-logic.ll | 264 + test/CodeGen/X86/fp128-calling-conv.ll | 47 + test/CodeGen/X86/fp128-cast.ll | 279 + test/CodeGen/X86/fp128-compare.ll | 96 + test/CodeGen/X86/fp128-i128.ll | 320 + test/CodeGen/X86/fp128-libcalls.ll | 107 + test/CodeGen/X86/fp128-load.ll | 35 + test/CodeGen/X86/fp128-store.ll | 14 + test/CodeGen/X86/fpcmp-soft-fp.ll | 127 + test/CodeGen/X86/fpstack-debuginstr-kill.ll | 16 +- test/CodeGen/X86/frameescape.ll | 128 - test/CodeGen/X86/frem-msvc32.ll | 12 + test/CodeGen/X86/funclet-layout.ll | 158 + test/CodeGen/X86/function-alias.ll | 12 + test/CodeGen/X86/gcc_except_table.ll | 2 +- test/CodeGen/X86/global-sections.ll | 7 +- test/CodeGen/X86/h-register-store.ll | 25 +- test/CodeGen/X86/h-registers-0.ll | 1 + test/CodeGen/X86/h-registers-1.ll | 1 + test/CodeGen/X86/h-registers-3.ll | 1 + test/CodeGen/X86/half.ll | 4 +- test/CodeGen/X86/hhvm-cc.ll | 241 + test/CodeGen/X86/i386-shrink-wrapping.ll | 113 + test/CodeGen/X86/immediate_merging.ll | 82 + test/CodeGen/X86/implicit-null-check.ll | 51 +- test/CodeGen/X86/imul.ll | 63 + test/CodeGen/X86/inalloca-stdcall.ll | 5 +- test/CodeGen/X86/inalloca.ll | 15 +- test/CodeGen/X86/inconsistent_landingpad.ll | 30 + test/CodeGen/X86/inline-asm-2addr.ll | 11 +- test/CodeGen/X86/inline-asm-sp-clobber-memcpy.ll | 2 +- test/CodeGen/X86/inline-sse.ll | 34 + test/CodeGen/X86/insertps-from-constantpool.ll | 20 + test/CodeGen/X86/insertps-unfold-load-bug.ll | 33 + test/CodeGen/X86/int-intrinsic.ll | 2 +- test/CodeGen/X86/late-address-taken.ll | 68 + test/CodeGen/X86/lea-opt.ll | 131 + test/CodeGen/X86/lit.local.cfg | 2 +- test/CodeGen/X86/localescape.ll | 143 + test/CodeGen/X86/lower-vec-shift-2.ll | 1 + test/CodeGen/X86/lsr-static-addr.ll | 2 +- test/CodeGen/X86/machine-combiner-int-vec.ll | 112 + test/CodeGen/X86/machine-combiner-int.ll | 194 + test/CodeGen/X86/machine-combiner.ll | 467 +- test/CodeGen/X86/machine-cp.ll | 38 +- test/CodeGen/X86/machine-trace-metrics-crash.ll | 4 +- test/CodeGen/X86/masked_gather_scatter.ll | 2012 ++- test/CodeGen/X86/masked_memop.ll | 524 +- test/CodeGen/X86/materialize.ll | 184 + test/CodeGen/X86/mcu-abi.ll | 112 + test/CodeGen/X86/memcpy-2.ll | 26 +- test/CodeGen/X86/memcpy.ll | 33 + .../X86/merge-store-partially-alias-loads.ll | 52 + .../X86/misched-code-difference-with-debug.ll | 12 +- test/CodeGen/X86/mmx-arg-passing-x86-64.ll | 1 + test/CodeGen/X86/mmx-arg-passing.ll | 1 + test/CodeGen/X86/mmx-coalescing.ll | 84 + test/CodeGen/X86/mmx-intrinsics.ll | 291 +- test/CodeGen/X86/mmx-only.ll | 21 + test/CodeGen/X86/movntdq-no-avx.ll | 2 +- test/CodeGen/X86/movpc32-check.ll | 42 + test/CodeGen/X86/movtopush.ll | 25 +- test/CodeGen/X86/mult-alt-x86.ll | 2 +- test/CodeGen/X86/musttail-varargs.ll | 43 + test/CodeGen/X86/nontemporal-2.ll | 21 +- test/CodeGen/X86/nontemporal.ll | 11 +- test/CodeGen/X86/null-streamer.ll | 4 +- test/CodeGen/X86/opt-ext-uses.ll | 8 +- test/CodeGen/X86/or-branch.ll | 30 +- test/CodeGen/X86/or-lea.ll | 120 + test/CodeGen/X86/palignr.ll | 1 + test/CodeGen/X86/patchpoint-verifiable.mir | 42 + test/CodeGen/X86/peephole-na-phys-copy-folding.ll | 190 + test/CodeGen/X86/pmul.ll | 297 +- test/CodeGen/X86/pop-stack-cleanup.ll | 76 + test/CodeGen/X86/powi.ll | 38 +- test/CodeGen/X86/pr11415.ll | 8 +- test/CodeGen/X86/pr11468.ll | 2 +- test/CodeGen/X86/pr11985.ll | 30 +- test/CodeGen/X86/pr13577.ll | 5 +- test/CodeGen/X86/pr15267.ll | 240 +- test/CodeGen/X86/pr17631.ll | 2 +- test/CodeGen/X86/pr21529.ll | 15 - test/CodeGen/X86/pr22019.ll | 2 +- test/CodeGen/X86/pr23900.ll | 29 - test/CodeGen/X86/pr24139.ll | 148 + test/CodeGen/X86/pr24602.ll | 17 + test/CodeGen/X86/pr25828.ll | 30 + test/CodeGen/X86/prolog-push-seq.ll | 19 + test/CodeGen/X86/pseudo_cmov_lower.ll | 267 + test/CodeGen/X86/pseudo_cmov_lower1.ll | 39 + test/CodeGen/X86/pseudo_cmov_lower2.ll | 100 + test/CodeGen/X86/psubus.ll | 580 +- test/CodeGen/X86/push-cfi-debug.ll | 53 + test/CodeGen/X86/push-cfi-obj.ll | 44 + test/CodeGen/X86/push-cfi.ll | 304 + test/CodeGen/X86/ragreedy-hoist-spill.ll | 2 +- test/CodeGen/X86/rem_crash.ll | 257 + test/CodeGen/X86/remat-invalid-liveness.ll | 85 - test/CodeGen/X86/rodata-relocs.ll | 8 +- test/CodeGen/X86/rounding-ops.ll | 24 +- test/CodeGen/X86/safestack.ll | 32 + test/CodeGen/X86/sar_fold.ll | 37 + test/CodeGen/X86/sar_fold64.ll | 43 + test/CodeGen/X86/scalar-fp-to-i64.ll | 151 + test/CodeGen/X86/scalar-int-to-fp.ll | 132 + test/CodeGen/X86/sdiv-pow2.ll | 33 + test/CodeGen/X86/seh-catch-all-win32.ll | 33 +- test/CodeGen/X86/seh-catch-all.ll | 29 +- test/CodeGen/X86/seh-catchpad.ll | 198 + test/CodeGen/X86/seh-except-finally.ll | 71 +- test/CodeGen/X86/seh-exception-code.ll | 38 + test/CodeGen/X86/seh-filter.ll | 21 - test/CodeGen/X86/seh-finally.ll | 50 +- test/CodeGen/X86/seh-safe-div-win32.ll | 42 +- test/CodeGen/X86/seh-safe-div.ll | 54 +- test/CodeGen/X86/seh-stack-realign-win32.ll | 99 - test/CodeGen/X86/seh-stack-realign.ll | 34 +- test/CodeGen/X86/setcc-lowering.ll | 1 + test/CodeGen/X86/setcc.ll | 20 + test/CodeGen/X86/shift-bmi2.ll | 20 +- test/CodeGen/X86/shrink-wrap-chkstk.ll | 37 + test/CodeGen/X86/slow-div.ll | 15 + test/CodeGen/X86/slow-unaligned-mem.ll | 95 + test/CodeGen/X86/soft-fp.ll | 34 +- test/CodeGen/X86/soft-sitofp.ll | 169 + test/CodeGen/X86/splat-for-size.ll | 197 +- test/CodeGen/X86/sqrt-fastmath.ll | 9 +- test/CodeGen/X86/sse-align-12.ll | 1 + test/CodeGen/X86/sse-minmax.ll | 2 +- test/CodeGen/X86/sse-only.ll | 20 + test/CodeGen/X86/sse-scalar-fp-arith-unary.ll | 1 + test/CodeGen/X86/sse2-vector-shifts.ll | 282 +- test/CodeGen/X86/sse2.ll | 1 + test/CodeGen/X86/sse3-avx-addsub-2.ll | 312 +- test/CodeGen/X86/sse3-avx-addsub.ll | 197 +- test/CodeGen/X86/sse3-intrinsics-fast-isel.ll | 171 + test/CodeGen/X86/sse3.ll | 7 +- test/CodeGen/X86/sse41-intrinsics-x86-upgrade.ll | 47 +- test/CodeGen/X86/sse41-intrinsics-x86.ll | 48 - test/CodeGen/X86/sse41-pmovxrm-intrinsics.ll | 185 +- test/CodeGen/X86/sse41.ll | 65 +- test/CodeGen/X86/sse4a-intrinsics-fast-isel.ll | 98 + test/CodeGen/X86/sse_partial_update.ll | 33 + test/CodeGen/X86/ssse3-intrinsics-fast-isel.ll | 290 + test/CodeGen/X86/stack-align-memcpy.ll | 2 +- test/CodeGen/X86/stack-folding-adx-x86_64.ll | 45 + test/CodeGen/X86/stack-folding-fp-avx1.ll | 18 +- test/CodeGen/X86/stack-folding-fp-sse42.ll | 28 +- test/CodeGen/X86/stack-folding-int-avx1.ll | 40 +- test/CodeGen/X86/stack-folding-int-avx2.ll | 55 +- test/CodeGen/X86/stack-folding-int-sse42.ll | 38 +- test/CodeGen/X86/stack-folding-mmx.ll | 148 +- test/CodeGen/X86/stack-folding-x86_64.ll | 2 +- test/CodeGen/X86/stack-folding-xop.ll | 2 +- test/CodeGen/X86/stack-probe-size.ll | 3 +- test/CodeGen/X86/stack-protector-dbginfo.ll | 36 +- test/CodeGen/X86/stack-protector-weight.ll | 4 +- test/CodeGen/X86/stackmap-frame-setup.ll | 20 + test/CodeGen/X86/statepoint-allocas.ll | 10 +- test/CodeGen/X86/statepoint-call-lowering.ll | 103 +- test/CodeGen/X86/statepoint-far-call.ll | 4 +- test/CodeGen/X86/statepoint-forward.ll | 16 +- .../X86/statepoint-gctransition-call-lowering.ll | 66 +- test/CodeGen/X86/statepoint-invoke.ll | 78 +- test/CodeGen/X86/statepoint-stack-usage.ll | 54 +- test/CodeGen/X86/statepoint-stackmap-format.ll | 96 +- test/CodeGen/X86/stdarg.ll | 10 +- test/CodeGen/X86/stores-merging.ll | 46 +- test/CodeGen/X86/switch-bt.ll | 8 +- test/CodeGen/X86/switch-edge-weight.ll | 281 + test/CodeGen/X86/switch-jump-table.ll | 54 +- test/CodeGen/X86/switch-order-weight.ll | 2 +- test/CodeGen/X86/switch.ll | 85 +- test/CodeGen/X86/swizzle-2.ll | 1 + test/CodeGen/X86/system-intrinsics-64-xsave.ll | 41 + test/CodeGen/X86/system-intrinsics-64-xsavec.ll | 21 + test/CodeGen/X86/system-intrinsics-64-xsaveopt.ll | 21 + test/CodeGen/X86/system-intrinsics-64-xsaves.ll | 41 + test/CodeGen/X86/system-intrinsics-64.ll | 2 +- test/CodeGen/X86/system-intrinsics-xsave.ll | 23 + test/CodeGen/X86/system-intrinsics-xsavec.ll | 12 + test/CodeGen/X86/system-intrinsics-xsaveopt.ll | 12 + test/CodeGen/X86/system-intrinsics-xsaves.ll | 23 + test/CodeGen/X86/system-intrinsics.ll | 2 +- test/CodeGen/X86/tail-dup-catchret.ll | 31 + test/CodeGen/X86/tail-merge-wineh.ll | 107 + test/CodeGen/X86/tail-opts.ll | 40 +- test/CodeGen/X86/tailcall-mem-intrinsics.ll | 4 +- test/CodeGen/X86/tailcall-msvc-conventions.ll | 189 + test/CodeGen/X86/tailcall-readnone.ll | 15 + test/CodeGen/X86/tls-android-negative.ll | 65 + test/CodeGen/X86/tls-android.ll | 89 + test/CodeGen/X86/tls-models.ll | 2 + test/CodeGen/X86/tls-pie.ll | 8 + test/CodeGen/X86/token_landingpad.ll | 21 + test/CodeGen/X86/trunc-store.ll | 49 + test/CodeGen/X86/unaligned-32-byte-memops.ll | 7 +- test/CodeGen/X86/unaligned-spill-folding.ll | 2 +- test/CodeGen/X86/unknown-location.ll | 8 +- test/CodeGen/X86/v2f32.ll | 1 + test/CodeGen/X86/vec_cast2.ll | 31 +- test/CodeGen/X86/vec_cmp_sint-128.ll | 722 + test/CodeGen/X86/vec_cmp_uint-128.ll | 860 + test/CodeGen/X86/vec_ctbits.ll | 129 +- test/CodeGen/X86/vec_extract-avx.ll | 114 +- test/CodeGen/X86/vec_fabs.ll | 2 +- test/CodeGen/X86/vec_fp_to_int.ll | 1269 +- test/CodeGen/X86/vec_insert-5.ll | 1 + test/CodeGen/X86/vec_int_to_fp.ll | 1920 ++- test/CodeGen/X86/vec_minmax_sint.ll | 2090 +++ test/CodeGen/X86/vec_minmax_uint.ll | 2229 +++ test/CodeGen/X86/vec_sdiv_to_shift.ll | 13 + test/CodeGen/X86/vec_trunc_sext.ll | 31 +- test/CodeGen/X86/vec_uint_to_fp-fastmath.ll | 130 + test/CodeGen/X86/vec_uint_to_fp.ll | 8 +- test/CodeGen/X86/vector-blend.ll | 72 +- test/CodeGen/X86/vector-idiv.ll | 1 + test/CodeGen/X86/vector-lzcnt-128.ll | 472 +- test/CodeGen/X86/vector-lzcnt-256.ll | 257 +- test/CodeGen/X86/vector-lzcnt-512.ll | 219 + .../CodeGen/X86/vector-merge-store-fp-constants.ll | 35 + test/CodeGen/X86/vector-popcnt-128.ll | 37 +- test/CodeGen/X86/vector-popcnt-256.ll | 73 +- test/CodeGen/X86/vector-popcnt-512.ll | 161 + test/CodeGen/X86/vector-rotate-128.ll | 1595 ++ test/CodeGen/X86/vector-rotate-256.ll | 1089 ++ test/CodeGen/X86/vector-sext.ll | 3988 ++++- test/CodeGen/X86/vector-shift-ashr-128.ll | 917 +- test/CodeGen/X86/vector-shift-ashr-256.ll | 691 +- test/CodeGen/X86/vector-shift-ashr-512.ll | 378 + test/CodeGen/X86/vector-shift-lshr-128.ll | 619 +- test/CodeGen/X86/vector-shift-lshr-256.ll | 444 +- test/CodeGen/X86/vector-shift-lshr-512.ll | 317 + test/CodeGen/X86/vector-shift-shl-128.ll | 501 +- test/CodeGen/X86/vector-shift-shl-256.ll | 403 +- test/CodeGen/X86/vector-shift-shl-512.ll | 293 + test/CodeGen/X86/vector-shuffle-128-v16.ll | 276 + test/CodeGen/X86/vector-shuffle-128-v2.ll | 318 +- test/CodeGen/X86/vector-shuffle-128-v4.ll | 92 + test/CodeGen/X86/vector-shuffle-128-v8.ll | 252 + test/CodeGen/X86/vector-shuffle-256-v16.ll | 249 +- test/CodeGen/X86/vector-shuffle-256-v32.ll | 210 +- test/CodeGen/X86/vector-shuffle-256-v4.ll | 703 +- test/CodeGen/X86/vector-shuffle-256-v8.ll | 221 +- test/CodeGen/X86/vector-shuffle-512-v16.ll | 134 + test/CodeGen/X86/vector-shuffle-512-v32.ll | 44 + test/CodeGen/X86/vector-shuffle-512-v8.ll | 2487 ++- test/CodeGen/X86/vector-shuffle-combining.ll | 1 + test/CodeGen/X86/vector-shuffle-mmx.ll | 1 + test/CodeGen/X86/vector-shuffle-sse1.ll | 1 + test/CodeGen/X86/vector-shuffle-sse4a.ll | 140 + test/CodeGen/X86/vector-shuffle-v1.ll | 439 + test/CodeGen/X86/vector-trunc.ll | 681 +- test/CodeGen/X86/vector-tzcnt-128.ll | 2035 +-- test/CodeGen/X86/vector-tzcnt-256.ll | 1455 +- test/CodeGen/X86/vector-tzcnt-512.ll | 271 + test/CodeGen/X86/vector-zext.ll | 1523 +- test/CodeGen/X86/vector-zmov.ll | 1 + ...gisters-cleared-in-machine-functions-liveins.ll | 19 + test/CodeGen/X86/vmovq.ll | 28 + test/CodeGen/X86/vselect-2.ll | 1 + test/CodeGen/X86/vselect-avx.ll | 12 +- test/CodeGen/X86/vselect-minmax.ll | 16332 +++++++++++++------ test/CodeGen/X86/vselect.ll | 1 + test/CodeGen/X86/vshift_scalar.ll | 1 + test/CodeGen/X86/wide-integer-cmp.ll | 130 + test/CodeGen/X86/widen_load-2.ll | 4 +- test/CodeGen/X86/widen_shuffle-1.ll | 1 + test/CodeGen/X86/win-catchpad-csrs.ll | 268 + test/CodeGen/X86/win-catchpad-nested-cxx.ll | 105 + test/CodeGen/X86/win-catchpad-nested.ll | 42 + test/CodeGen/X86/win-catchpad-varargs.ll | 101 + test/CodeGen/X86/win-catchpad.ll | 353 + test/CodeGen/X86/win-cleanuppad.ll | 199 + test/CodeGen/X86/win-funclet-cfi.ll | 95 + test/CodeGen/X86/win-mixed-ehpersonality.ll | 81 + test/CodeGen/X86/win32-eh-states.ll | 213 +- test/CodeGen/X86/win32-eh.ll | 49 +- test/CodeGen/X86/win32-pic-jumptable.ll | 8 +- test/CodeGen/X86/win32-seh-catchpad-realign.ll | 77 + test/CodeGen/X86/win32-seh-catchpad.ll | 231 + test/CodeGen/X86/win32-seh-nested-finally.ll | 80 + test/CodeGen/X86/win32-spill-xmm.ll | 40 + test/CodeGen/X86/win64_frame.ll | 70 +- test/CodeGen/X86/win64_sibcall.ll | 38 + test/CodeGen/X86/win_coreclr_chkstk.ll | 143 + test/CodeGen/X86/win_eh_prepare.ll | 82 - test/CodeGen/X86/win_ftol2.ll | 166 - test/CodeGen/X86/wineh-coreclr.ll | 267 + test/CodeGen/X86/wineh-exceptionpointer.ll | 26 + test/CodeGen/X86/wineh-no-ehpads.ll | 20 + test/CodeGen/X86/x32-function_pointer-3.ll | 2 +- test/CodeGen/X86/x32-indirectbr.ll | 26 + test/CodeGen/X86/x32-landingpad.ll | 27 + test/CodeGen/X86/x32-va_start.ll | 99 + test/CodeGen/X86/x86-32-intrcc.ll | 79 + test/CodeGen/X86/x86-64-baseptr.ll | 4 +- .../X86/x86-64-double-precision-shift-left.ll | 17 +- .../X86/x86-64-double-precision-shift-right.ll | 9 +- test/CodeGen/X86/x86-64-double-shifts-Oz-Os-O2.ll | 4 +- test/CodeGen/X86/x86-64-intrcc.ll | 86 + test/CodeGen/X86/x86-64-ms_abi-vararg.ll | 108 + test/CodeGen/X86/x86-64-pic-10.ll | 2 +- test/CodeGen/X86/x86-fold-pshufb.ll | 20 +- test/CodeGen/X86/x86-sanitizer-shrink-wrapping.ll | 40 + test/CodeGen/X86/x86-setcc-int-to-fp-combine.ll | 16 +- test/CodeGen/X86/x86-shrink-wrap-unwind.ll | 153 + test/CodeGen/X86/x86-shrink-wrapping.ll | 254 +- test/CodeGen/X86/x86-win64-shrink-wrapping.ll | 126 + test/CodeGen/X86/xop-intrinsics-x86_64.ll | 33 +- test/CodeGen/X86/xop-pcmov.ll | 163 + 495 files changed, 85737 insertions(+), 19006 deletions(-) create mode 100644 test/CodeGen/X86/add-nsw-sext.ll create mode 100644 test/CodeGen/X86/and-encoding.ll create mode 100644 test/CodeGen/X86/atomic-flags.ll create mode 100644 test/CodeGen/X86/atomic-non-integer.ll create mode 100644 test/CodeGen/X86/avg.ll create mode 100644 test/CodeGen/X86/avx-isa-check.ll create mode 100644 test/CodeGen/X86/avx512-bugfix-25270.ll create mode 100644 test/CodeGen/X86/avx512-ext.ll create mode 100644 test/CodeGen/X86/avx512-extract-subvector.ll create mode 100644 test/CodeGen/X86/avx512-skx-insert-subvec.ll delete mode 100644 test/CodeGen/X86/avx512-trunc-ext.ll create mode 100644 test/CodeGen/X86/avx512-trunc.ll create mode 100644 test/CodeGen/X86/avx512cd-intrinsics.ll create mode 100644 test/CodeGen/X86/avx512cdvl-intrinsics.ll create mode 100644 test/CodeGen/X86/avx512dq-intrinsics.ll create mode 100644 test/CodeGen/X86/bit-piece-comment.ll create mode 100644 test/CodeGen/X86/bitreverse.ll create mode 100644 test/CodeGen/X86/branchfolding-catchpads.ll create mode 100644 test/CodeGen/X86/catchpad-realign-savexmm.ll create mode 100644 test/CodeGen/X86/catchpad-regmask.ll create mode 100644 test/CodeGen/X86/catchpad-weight.ll create mode 100644 test/CodeGen/X86/catchret-empty-fallthrough.ll create mode 100644 test/CodeGen/X86/catchret-fallthrough.ll create mode 100644 test/CodeGen/X86/cleanuppad-inalloca.ll create mode 100644 test/CodeGen/X86/cleanuppad-large-codemodel.ll create mode 100644 test/CodeGen/X86/cleanuppad-realign.ll create mode 100644 test/CodeGen/X86/coal-sections.ll create mode 100644 test/CodeGen/X86/coalescer-win64.ll create mode 100644 test/CodeGen/X86/code_placement_cold_loop_blocks.ll create mode 100644 test/CodeGen/X86/code_placement_ignore_succ_in_inner_loop.ll create mode 100644 test/CodeGen/X86/code_placement_loop_rotation.ll create mode 100644 test/CodeGen/X86/code_placement_loop_rotation2.ll create mode 100644 test/CodeGen/X86/combine-multiplies.ll delete mode 100644 test/CodeGen/X86/combine-sse2-intrinsics.ll create mode 100644 test/CodeGen/X86/constant-hoisting-and.ll create mode 100644 test/CodeGen/X86/constant-hoisting-cmp.ll delete mode 100644 test/CodeGen/X86/cppeh-nounwind.ll create mode 100644 test/CodeGen/X86/cxx_tlscc64.ll create mode 100644 test/CodeGen/X86/dag-fmf-cse.ll create mode 100644 test/CodeGen/X86/dag-merge-fast-accesses.ll create mode 100644 test/CodeGen/X86/darwin-tls.ll create mode 100644 test/CodeGen/X86/debugloc-argsize.ll create mode 100644 test/CodeGen/X86/eh-null-personality.ll create mode 100644 test/CodeGen/X86/emutls-pic.ll create mode 100644 test/CodeGen/X86/emutls-pie.ll create mode 100644 test/CodeGen/X86/emutls.ll create mode 100644 test/CodeGen/X86/emutls_generic.ll create mode 100644 test/CodeGen/X86/expand-vr64-gr64-copy.mir create mode 100644 test/CodeGen/X86/extractelement-legalization-cycle.ll create mode 100644 test/CodeGen/X86/fadd-combines.ll create mode 100644 test/CodeGen/X86/fast-isel-bitcasts-avx.ll create mode 100644 test/CodeGen/X86/fast-isel-bitcasts.ll create mode 100644 test/CodeGen/X86/fast-isel-deadcode.ll create mode 100644 test/CodeGen/X86/fast-isel-emutls.ll create mode 100644 test/CodeGen/X86/fast-isel-nontemporal.ll create mode 100644 test/CodeGen/X86/fast-isel-stackcheck.ll create mode 100644 test/CodeGen/X86/fixup-lea.ll create mode 100644 test/CodeGen/X86/fma-commute-x86.ll create mode 100644 test/CodeGen/X86/fma-scalar-memfold.ll create mode 100644 test/CodeGen/X86/fold-push.ll create mode 100644 test/CodeGen/X86/fp-logic.ll create mode 100644 test/CodeGen/X86/fp128-calling-conv.ll create mode 100644 test/CodeGen/X86/fp128-cast.ll create mode 100644 test/CodeGen/X86/fp128-compare.ll create mode 100644 test/CodeGen/X86/fp128-i128.ll create mode 100644 test/CodeGen/X86/fp128-libcalls.ll create mode 100644 test/CodeGen/X86/fp128-load.ll create mode 100644 test/CodeGen/X86/fp128-store.ll create mode 100644 test/CodeGen/X86/fpcmp-soft-fp.ll delete mode 100644 test/CodeGen/X86/frameescape.ll create mode 100644 test/CodeGen/X86/frem-msvc32.ll create mode 100644 test/CodeGen/X86/funclet-layout.ll create mode 100644 test/CodeGen/X86/function-alias.ll create mode 100644 test/CodeGen/X86/hhvm-cc.ll create mode 100644 test/CodeGen/X86/i386-shrink-wrapping.ll create mode 100644 test/CodeGen/X86/immediate_merging.ll create mode 100644 test/CodeGen/X86/inconsistent_landingpad.ll create mode 100644 test/CodeGen/X86/inline-sse.ll create mode 100644 test/CodeGen/X86/insertps-from-constantpool.ll create mode 100644 test/CodeGen/X86/insertps-unfold-load-bug.ll create mode 100644 test/CodeGen/X86/late-address-taken.ll create mode 100644 test/CodeGen/X86/lea-opt.ll create mode 100644 test/CodeGen/X86/localescape.ll create mode 100644 test/CodeGen/X86/machine-combiner-int-vec.ll create mode 100644 test/CodeGen/X86/machine-combiner-int.ll create mode 100644 test/CodeGen/X86/materialize.ll create mode 100644 test/CodeGen/X86/mcu-abi.ll create mode 100644 test/CodeGen/X86/merge-store-partially-alias-loads.ll create mode 100644 test/CodeGen/X86/mmx-coalescing.ll create mode 100644 test/CodeGen/X86/mmx-only.ll create mode 100644 test/CodeGen/X86/movpc32-check.ll create mode 100644 test/CodeGen/X86/or-lea.ll create mode 100644 test/CodeGen/X86/patchpoint-verifiable.mir create mode 100644 test/CodeGen/X86/peephole-na-phys-copy-folding.ll create mode 100644 test/CodeGen/X86/pop-stack-cleanup.ll delete mode 100644 test/CodeGen/X86/pr21529.ll delete mode 100644 test/CodeGen/X86/pr23900.ll create mode 100644 test/CodeGen/X86/pr24139.ll create mode 100644 test/CodeGen/X86/pr24602.ll create mode 100644 test/CodeGen/X86/pr25828.ll create mode 100644 test/CodeGen/X86/prolog-push-seq.ll create mode 100644 test/CodeGen/X86/pseudo_cmov_lower.ll create mode 100644 test/CodeGen/X86/pseudo_cmov_lower1.ll create mode 100644 test/CodeGen/X86/pseudo_cmov_lower2.ll create mode 100644 test/CodeGen/X86/push-cfi-debug.ll create mode 100644 test/CodeGen/X86/push-cfi-obj.ll create mode 100644 test/CodeGen/X86/push-cfi.ll create mode 100644 test/CodeGen/X86/rem_crash.ll delete mode 100644 test/CodeGen/X86/remat-invalid-liveness.ll create mode 100644 test/CodeGen/X86/safestack.ll create mode 100644 test/CodeGen/X86/sar_fold.ll create mode 100644 test/CodeGen/X86/sar_fold64.ll create mode 100644 test/CodeGen/X86/scalar-fp-to-i64.ll create mode 100644 test/CodeGen/X86/scalar-int-to-fp.ll create mode 100644 test/CodeGen/X86/sdiv-pow2.ll create mode 100644 test/CodeGen/X86/seh-catchpad.ll create mode 100644 test/CodeGen/X86/seh-exception-code.ll delete mode 100644 test/CodeGen/X86/seh-filter.ll delete mode 100644 test/CodeGen/X86/seh-stack-realign-win32.ll create mode 100644 test/CodeGen/X86/shrink-wrap-chkstk.ll create mode 100644 test/CodeGen/X86/slow-unaligned-mem.ll create mode 100644 test/CodeGen/X86/soft-sitofp.ll create mode 100644 test/CodeGen/X86/sse-only.ll create mode 100644 test/CodeGen/X86/sse3-intrinsics-fast-isel.ll create mode 100644 test/CodeGen/X86/sse4a-intrinsics-fast-isel.ll create mode 100644 test/CodeGen/X86/ssse3-intrinsics-fast-isel.ll create mode 100644 test/CodeGen/X86/stack-folding-adx-x86_64.ll create mode 100644 test/CodeGen/X86/stackmap-frame-setup.ll create mode 100644 test/CodeGen/X86/switch-edge-weight.ll create mode 100644 test/CodeGen/X86/system-intrinsics-64-xsave.ll create mode 100644 test/CodeGen/X86/system-intrinsics-64-xsavec.ll create mode 100644 test/CodeGen/X86/system-intrinsics-64-xsaveopt.ll create mode 100644 test/CodeGen/X86/system-intrinsics-64-xsaves.ll create mode 100644 test/CodeGen/X86/system-intrinsics-xsave.ll create mode 100644 test/CodeGen/X86/system-intrinsics-xsavec.ll create mode 100644 test/CodeGen/X86/system-intrinsics-xsaveopt.ll create mode 100644 test/CodeGen/X86/system-intrinsics-xsaves.ll create mode 100644 test/CodeGen/X86/tail-dup-catchret.ll create mode 100644 test/CodeGen/X86/tail-merge-wineh.ll create mode 100644 test/CodeGen/X86/tailcall-msvc-conventions.ll create mode 100644 test/CodeGen/X86/tailcall-readnone.ll create mode 100644 test/CodeGen/X86/tls-android-negative.ll create mode 100644 test/CodeGen/X86/tls-android.ll create mode 100644 test/CodeGen/X86/token_landingpad.ll create mode 100644 test/CodeGen/X86/trunc-store.ll create mode 100644 test/CodeGen/X86/vec_cmp_sint-128.ll create mode 100644 test/CodeGen/X86/vec_cmp_uint-128.ll create mode 100644 test/CodeGen/X86/vec_minmax_sint.ll create mode 100644 test/CodeGen/X86/vec_minmax_uint.ll create mode 100644 test/CodeGen/X86/vec_uint_to_fp-fastmath.ll create mode 100644 test/CodeGen/X86/vector-lzcnt-512.ll create mode 100644 test/CodeGen/X86/vector-merge-store-fp-constants.ll create mode 100644 test/CodeGen/X86/vector-popcnt-512.ll create mode 100644 test/CodeGen/X86/vector-rotate-128.ll create mode 100644 test/CodeGen/X86/vector-rotate-256.ll create mode 100644 test/CodeGen/X86/vector-shift-ashr-512.ll create mode 100644 test/CodeGen/X86/vector-shift-lshr-512.ll create mode 100644 test/CodeGen/X86/vector-shift-shl-512.ll create mode 100644 test/CodeGen/X86/vector-shuffle-512-v32.ll create mode 100644 test/CodeGen/X86/vector-shuffle-v1.ll create mode 100644 test/CodeGen/X86/vector-tzcnt-512.ll create mode 100644 test/CodeGen/X86/virtual-registers-cleared-in-machine-functions-liveins.ll create mode 100644 test/CodeGen/X86/vmovq.ll create mode 100644 test/CodeGen/X86/wide-integer-cmp.ll create mode 100644 test/CodeGen/X86/win-catchpad-csrs.ll create mode 100644 test/CodeGen/X86/win-catchpad-nested-cxx.ll create mode 100644 test/CodeGen/X86/win-catchpad-nested.ll create mode 100644 test/CodeGen/X86/win-catchpad-varargs.ll create mode 100644 test/CodeGen/X86/win-catchpad.ll create mode 100644 test/CodeGen/X86/win-cleanuppad.ll create mode 100644 test/CodeGen/X86/win-funclet-cfi.ll create mode 100644 test/CodeGen/X86/win-mixed-ehpersonality.ll create mode 100644 test/CodeGen/X86/win32-seh-catchpad-realign.ll create mode 100644 test/CodeGen/X86/win32-seh-catchpad.ll create mode 100644 test/CodeGen/X86/win32-seh-nested-finally.ll create mode 100644 test/CodeGen/X86/win32-spill-xmm.ll create mode 100644 test/CodeGen/X86/win64_sibcall.ll create mode 100644 test/CodeGen/X86/win_coreclr_chkstk.ll delete mode 100644 test/CodeGen/X86/win_eh_prepare.ll delete mode 100644 test/CodeGen/X86/win_ftol2.ll create mode 100644 test/CodeGen/X86/wineh-coreclr.ll create mode 100644 test/CodeGen/X86/wineh-exceptionpointer.ll create mode 100644 test/CodeGen/X86/wineh-no-ehpads.ll create mode 100644 test/CodeGen/X86/x32-indirectbr.ll create mode 100644 test/CodeGen/X86/x32-landingpad.ll create mode 100644 test/CodeGen/X86/x32-va_start.ll create mode 100644 test/CodeGen/X86/x86-32-intrcc.ll create mode 100644 test/CodeGen/X86/x86-64-intrcc.ll create mode 100644 test/CodeGen/X86/x86-64-ms_abi-vararg.ll create mode 100644 test/CodeGen/X86/x86-sanitizer-shrink-wrapping.ll create mode 100644 test/CodeGen/X86/x86-shrink-wrap-unwind.ll create mode 100644 test/CodeGen/X86/x86-win64-shrink-wrapping.ll create mode 100644 test/CodeGen/X86/xop-pcmov.ll (limited to 'test/CodeGen/X86') diff --git a/test/CodeGen/X86/2006-10-02-BoolRetCrash.ll b/test/CodeGen/X86/2006-10-02-BoolRetCrash.ll index 795d4647a3f6..609dbc155ed9 100644 --- a/test/CodeGen/X86/2006-10-02-BoolRetCrash.ll +++ b/test/CodeGen/X86/2006-10-02-BoolRetCrash.ll @@ -1,5 +1,6 @@ ; RUN: llc < %s ; PR933 +; REQUIRES: default_triple define fastcc i1 @test() { ret i1 true diff --git a/test/CodeGen/X86/2006-10-19-SwitchUnnecessaryBranching.ll b/test/CodeGen/X86/2006-10-19-SwitchUnnecessaryBranching.ll index dd670648daf6..332816e22cda 100644 --- a/test/CodeGen/X86/2006-10-19-SwitchUnnecessaryBranching.ll +++ b/test/CodeGen/X86/2006-10-19-SwitchUnnecessaryBranching.ll @@ -6,8 +6,8 @@ define i32 @test(i32 %argc, i8** %argv) nounwind { entry: ; CHECK: cmpl $2 -; CHECK-NEXT: jne -; CHECK-NEXT: %bb2 +; CHECK-NEXT: je +; CHECK-NEXT: %entry switch i32 %argc, label %UnifiedReturnBlock [ i32 1, label %bb diff --git a/test/CodeGen/X86/2007-09-06-ExtWeakAliasee.ll b/test/CodeGen/X86/2007-09-06-ExtWeakAliasee.ll index 62c503da35a6..65b577b1e7d7 100644 --- a/test/CodeGen/X86/2007-09-06-ExtWeakAliasee.ll +++ b/test/CodeGen/X86/2007-09-06-ExtWeakAliasee.ll @@ -1,6 +1,6 @@ ; RUN: llc < %s -mtriple=i686-pc-linux-gnu | FileCheck %s -@__gthrw_pthread_once = weak alias i32 (i32*, void ()*)* @pthread_once ; [#uses=0] +@__gthrw_pthread_once = weak alias i32 (i32*, void ()*), i32 (i32*, void ()*)* @pthread_once ; [#uses=0] define weak i32 @pthread_once(i32*, void ()*) { ret i32 0 diff --git a/test/CodeGen/X86/2008-03-12-ThreadLocalAlias.ll b/test/CodeGen/X86/2008-03-12-ThreadLocalAlias.ll index a9e3f33ec618..2ca003e052aa 100644 --- a/test/CodeGen/X86/2008-03-12-ThreadLocalAlias.ll +++ b/test/CodeGen/X86/2008-03-12-ThreadLocalAlias.ll @@ -8,7 +8,7 @@ target triple = "i386-pc-linux-gnu" @__resp = thread_local global %struct.__res_state* @_res ; <%struct.__res_state**> [#uses=1] @_res = global %struct.__res_state zeroinitializer, section ".bss" ; <%struct.__res_state*> [#uses=1] -@__libc_resp = hidden thread_local alias %struct.__res_state** @__resp ; <%struct.__res_state**> [#uses=2] +@__libc_resp = hidden thread_local alias %struct.__res_state*, %struct.__res_state** @__resp ; <%struct.__res_state**> [#uses=2] define i32 @foo() { ; CHECK-LABEL: foo: diff --git a/test/CodeGen/X86/2008-03-14-SpillerCrash.ll b/test/CodeGen/X86/2008-03-14-SpillerCrash.ll index d60d0c2fb0bc..d484b45a5763 100644 --- a/test/CodeGen/X86/2008-03-14-SpillerCrash.ll +++ b/test/CodeGen/X86/2008-03-14-SpillerCrash.ll @@ -6,7 +6,7 @@ %struct.locale_data = type { i8*, i8*, i32, i32, { void (%struct.locale_data*)*, %struct.anon }, i32, i32, i32, [0 x %struct.locale_data_value] } %struct.locale_data_value = type { i32* } -@wcstoll_l = alias i64 (i32*, i32**, i32, %struct.__locale_struct*)* @__wcstoll_l +@wcstoll_l = alias i64 (i32*, i32**, i32, %struct.__locale_struct*), i64 (i32*, i32**, i32, %struct.__locale_struct*)* @__wcstoll_l define i64 @____wcstoll_l_internal(i32* %nptr, i32** %endptr, i32 %base, i32 %group, %struct.__locale_struct* %loc) nounwind { entry: diff --git a/test/CodeGen/X86/2008-06-13-NotVolatileLoadStore.ll b/test/CodeGen/X86/2008-06-13-NotVolatileLoadStore.ll index 422d68e7ff49..de95e7925f08 100644 --- a/test/CodeGen/X86/2008-06-13-NotVolatileLoadStore.ll +++ b/test/CodeGen/X86/2008-06-13-NotVolatileLoadStore.ll @@ -1,8 +1,10 @@ -; RUN: llc < %s -march=x86 | not grep movsd -; RUN: llc < %s -march=x86 | grep movw -; RUN: llc < %s -march=x86 | grep addw +; RUN: llc < %s -march=x86 | FileCheck %s ; These transforms are turned off for load volatiles and stores. ; Check that they weren't turned off for all loads and stores! +; CHECK-LABEL: f: +; CHECK-NOT: movsd +; CHECK: movw +; CHECK: addw @atomic = global double 0.000000e+00 ; [#uses=1] @atomic2 = global double 0.000000e+00 ; [#uses=1] diff --git a/test/CodeGen/X86/2008-09-05-sinttofp-2xi32.ll b/test/CodeGen/X86/2008-09-05-sinttofp-2xi32.ll index 757f1ff68253..84d373d70a2d 100644 --- a/test/CodeGen/X86/2008-09-05-sinttofp-2xi32.ll +++ b/test/CodeGen/X86/2008-09-05-sinttofp-2xi32.ll @@ -1,18 +1,19 @@ -; RUN: llc < %s -march=x86 -mattr=+sse2 -mattr=+mmx | not grep unpcklpd -; RUN: llc < %s -march=x86 -mattr=+sse2 -mattr=+mmx | not grep unpckhpd -; RUN: llc < %s -march=x86 -mattr=+sse2 | grep cvttpd2pi | count 1 -; RUN: llc < %s -march=x86 -mattr=+sse2 | grep cvtpi2pd | count 1 +; RUN: llc < %s -march=x86 -mattr=+sse2,+mmx | FileCheck %s ; originally from PR2687, but things don't work that way any more. ; there are no MMX instructions here; we use XMM. define <2 x double> @a(<2 x i32> %x) nounwind { entry: +; CHECK-LABEL: a +; CHECK-NOT: unpcklpd %y = sitofp <2 x i32> %x to <2 x double> ret <2 x double> %y } define <2 x i32> @b(<2 x double> %x) nounwind { entry: +; CHECK-LABEL: b +; CHECK-NOT: unpckhpd %y = fptosi <2 x double> %x to <2 x i32> ret <2 x i32> %y } @@ -21,12 +22,18 @@ entry: define <2 x double> @a2(x86_mmx %x) nounwind { entry: +; CHECK-LABEL: a2 +; CHECK: cvtpi2pd +; CHECK-NOT: cvtpi2pd %y = tail call <2 x double> @llvm.x86.sse.cvtpi2pd(x86_mmx %x) ret <2 x double> %y } define x86_mmx @b2(<2 x double> %x) nounwind { entry: +; CHECK-LABEL: b2 +; CHECK: cvttpd2pi +; CHECK-NOT: cvttpd2pi %y = tail call x86_mmx @llvm.x86.sse.cvttpd2pi (<2 x double> %x) ret x86_mmx %y } diff --git a/test/CodeGen/X86/2009-02-12-DebugInfoVLA.ll b/test/CodeGen/X86/2009-02-12-DebugInfoVLA.ll index 6c177e5b5f5a..2abb5ba7cd52 100644 --- a/test/CodeGen/X86/2009-02-12-DebugInfoVLA.ll +++ b/test/CodeGen/X86/2009-02-12-DebugInfoVLA.ll @@ -76,15 +76,15 @@ declare i64 @strlen(i8*) nounwind readonly declare void @llvm.stackrestore(i8*) nounwind -!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "s1", line: 2, arg: 0, scope: !1, file: !2, type: !6) -!1 = !DISubprogram(name: "foo", linkageName: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !2, type: !3) -!2 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !17, enums: !18, retainedTypes: !18) +!0 = !DILocalVariable(name: "s1", line: 2, arg: 1, scope: !1, file: !2, type: !6) +!1 = distinct !DISubprogram(name: "foo", linkageName: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !2, type: !3) +!2 = distinct !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !17, enums: !18, retainedTypes: !18) !3 = !DISubroutineType(types: !4) !4 = !{!5, !6} !5 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char) !6 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !2, baseType: !5) !7 = !DILocation(line: 2, scope: !1) -!8 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "str.0", line: 3, scope: !1, file: !2, type: !9) +!8 = !DILocalVariable(name: "str.0", line: 3, scope: !1, file: !2, type: !9) !9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, scope: !2, baseType: !10) !10 = !DICompositeType(tag: DW_TAG_array_type, size: 8, align: 8, scope: !2, baseType: !5, elements: !11) !11 = !{!12} diff --git a/test/CodeGen/X86/2009-05-23-dagcombine-shifts.ll b/test/CodeGen/X86/2009-05-23-dagcombine-shifts.ll index 7c87598d0d9c..609be3bb2e54 100644 --- a/test/CodeGen/X86/2009-05-23-dagcombine-shifts.ll +++ b/test/CodeGen/X86/2009-05-23-dagcombine-shifts.ll @@ -4,15 +4,23 @@ ; a shr (X, -8) that gets subsequently "optimized away" as undef ; PR4254 +; after fixing PR24373 +; shlq $56, %rdi +; sarq $48, %rdi +; folds into +; movsbq %dil, %rax +; shlq $8, %rax +; which is better for x86 + target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128" target triple = "x86_64-unknown-linux-gnu" define i64 @foo(i64 %b) nounwind readnone { entry: ; CHECK-LABEL: foo: -; CHECK: shlq $56, %rdi -; CHECK: sarq $48, %rdi -; CHECK: leaq 1(%rdi), %rax +; CHECK: movsbq %dil, %rax +; CHECK: shlq $8, %rax +; CHECK: orq $1, %rax %shl = shl i64 %b, 56 ; [#uses=1] %shr = ashr i64 %shl, 48 ; [#uses=1] %add5 = or i64 %shr, 1 ; [#uses=1] diff --git a/test/CodeGen/X86/2009-06-03-Win64SpillXMM.ll b/test/CodeGen/X86/2009-06-03-Win64SpillXMM.ll index dfb98bb1ab39..a74aa2dd4623 100644 --- a/test/CodeGen/X86/2009-06-03-Win64SpillXMM.ll +++ b/test/CodeGen/X86/2009-06-03-Win64SpillXMM.ll @@ -1,7 +1,9 @@ ; RUN: llc -mcpu=generic -mtriple=x86_64-mingw32 < %s | FileCheck %s -; CHECK: subq $40, %rsp -; CHECK: movaps %xmm8, 16(%rsp) -; CHECK: movaps %xmm7, (%rsp) +; CHECK: pushq %rbp +; CHECK: subq $32, %rsp +; CHECK: leaq 32(%rsp), %rbp +; CHECK: movaps %xmm8, -16(%rbp) +; CHECK: movaps %xmm7, -32(%rbp) define i32 @a() nounwind { entry: diff --git a/test/CodeGen/X86/2009-06-05-VariableIndexInsert.ll b/test/CodeGen/X86/2009-06-05-VariableIndexInsert.ll index 8bb3dc63a3b9..71a560a63ec5 100644 --- a/test/CodeGen/X86/2009-06-05-VariableIndexInsert.ll +++ b/test/CodeGen/X86/2009-06-05-VariableIndexInsert.ll @@ -1,4 +1,5 @@ ; RUN: llc < %s +; REQUIRES: default_triple define <2 x i64> @_mm_insert_epi16(<2 x i64> %a, i32 %b, i32 %imm) nounwind readnone { entry: diff --git a/test/CodeGen/X86/2009-06-06-ConcatVectors.ll b/test/CodeGen/X86/2009-06-06-ConcatVectors.ll index 92419fcb8b81..e26a8608a496 100644 --- a/test/CodeGen/X86/2009-06-06-ConcatVectors.ll +++ b/test/CodeGen/X86/2009-06-06-ConcatVectors.ll @@ -1,4 +1,5 @@ ; RUN: llc < %s +; REQUIRES: default_triple define <2 x i64> @_mm_movpi64_pi64(<1 x i64> %a, <1 x i64> %b) nounwind readnone { entry: diff --git a/test/CodeGen/X86/2009-10-16-Scope.ll b/test/CodeGen/X86/2009-10-16-Scope.ll index bda7340b3643..06a56ad90205 100644 --- a/test/CodeGen/X86/2009-10-16-Scope.ll +++ b/test/CodeGen/X86/2009-10-16-Scope.ll @@ -24,9 +24,9 @@ declare i32 @foo(i32) ssp !0 = !DILocation(line: 5, column: 2, scope: !1) !1 = distinct !DILexicalBlock(line: 1, column: 1, file: null, scope: !2) -!2 = !DISubprogram(name: "bar", linkageName: "bar", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !3) -!3 = !DICompileUnit(language: DW_LANG_C99, producer: "clang 1.1", isOptimized: true, emissionKind: 0, file: !8, retainedTypes: !9) -!4 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "count_", line: 5, scope: !5, file: !3, type: !6) +!2 = distinct !DISubprogram(name: "bar", linkageName: "bar", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !3) +!3 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang 1.1", isOptimized: true, emissionKind: 0, file: !8, retainedTypes: !9) +!4 = !DILocalVariable(name: "count_", line: 5, scope: !5, file: !3, type: !6) !5 = distinct !DILexicalBlock(line: 1, column: 1, file: null, scope: !1) !6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !7 = !DILocation(line: 6, column: 1, scope: !2) diff --git a/test/CodeGen/X86/2010-01-18-DbgValue.ll b/test/CodeGen/X86/2010-01-18-DbgValue.ll index db56ae65d51e..c15e7a79bfa1 100644 --- a/test/CodeGen/X86/2010-01-18-DbgValue.ll +++ b/test/CodeGen/X86/2010-01-18-DbgValue.ll @@ -6,7 +6,7 @@ %struct.Pt = type { double, double } %struct.Rect = type { %struct.Pt, %struct.Pt } -define double @foo(%struct.Rect* byval %my_r0) nounwind ssp { +define double @foo(%struct.Rect* byval %my_r0) nounwind ssp !dbg !1 { entry: ;CHECK: DEBUG_VALUE %retval = alloca double ; [#uses=2] @@ -31,10 +31,10 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone !llvm.dbg.cu = !{!3} !llvm.module.flags = !{!21} -!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "my_r0", line: 11, arg: 0, scope: !1, file: !2, type: !7) -!1 = !DISubprogram(name: "foo", linkageName: "foo", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 11, file: !19, scope: !2, type: !4, function: double (%struct.Rect*)* @foo) +!0 = !DILocalVariable(name: "my_r0", line: 11, arg: 1, scope: !1, file: !2, type: !7) +!1 = distinct !DISubprogram(name: "foo", linkageName: "foo", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 11, file: !19, scope: !2, type: !4) !2 = !DIFile(filename: "b2.c", directory: "/tmp/") -!3 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !19, enums: !20, retainedTypes: !20, subprograms: !18) +!3 = distinct !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 0, file: !19, enums: !20, retainedTypes: !20, subprograms: !18) !4 = !DISubroutineType(types: !5) !5 = !{!6, !7} !6 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float) diff --git a/test/CodeGen/X86/2010-02-01-DbgValueCrash.ll b/test/CodeGen/X86/2010-02-01-DbgValueCrash.ll index e65edac86ecc..eb077c074bc2 100644 --- a/test/CodeGen/X86/2010-02-01-DbgValueCrash.ll +++ b/test/CodeGen/X86/2010-02-01-DbgValueCrash.ll @@ -1,4 +1,6 @@ ; RUN: llc -O1 < %s +; REQUIRES: default_triple + ; ModuleID = 'pr6157.bc' ; formerly crashed in SelectionDAGBuilder @@ -16,7 +18,7 @@ entry: declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnone declare void @"tart.reflect.ComplexType.create->tart.core.Object"(%tart.reflect.ComplexType*) nounwind readnone -!0 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !15, enums: !16, retainedTypes: !16) +!0 = distinct !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 0, file: !15, enums: !16, retainedTypes: !16) !1 = !DIDerivedType(tag: DW_TAG_const_type, size: 192, align: 64, file: !15, scope: !0, baseType: !2) !2 = !DICompositeType(tag: DW_TAG_structure_type, name: "C", line: 1, size: 192, align: 64, file: !15, scope: !0, elements: !3) !3 = !{!4, !6, !7} @@ -24,9 +26,9 @@ declare void @"tart.reflect.ComplexType.create->tart.core.Object"(%tart.reflect. !5 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float) !6 = !DIDerivedType(tag: DW_TAG_member, name: "y", line: 1, size: 64, align: 64, offset: 64, file: !15, scope: !2, baseType: !5) !7 = !DIDerivedType(tag: DW_TAG_member, name: "z", line: 1, size: 64, align: 64, offset: 128, file: !15, scope: !2, baseType: !5) -!8 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "t", line: 5, scope: !9, file: !0, type: !2) +!8 = !DILocalVariable(name: "t", line: 5, scope: !9, file: !0, type: !2) !9 = distinct !DILexicalBlock(line: 0, column: 0, file: null, scope: !10) -!10 = !DISubprogram(name: "foo", linkageName: "foo", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !0, type: !11) +!10 = distinct !DISubprogram(name: "foo", linkageName: "foo", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !0, type: !11) !11 = !DISubroutineType(types: !12) !12 = !{!13} !13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) diff --git a/test/CodeGen/X86/2010-05-25-DotDebugLoc.ll b/test/CodeGen/X86/2010-05-25-DotDebugLoc.ll index 3b99e91915f0..f157d5011b02 100644 --- a/test/CodeGen/X86/2010-05-25-DotDebugLoc.ll +++ b/test/CodeGen/X86/2010-05-25-DotDebugLoc.ll @@ -7,7 +7,7 @@ %0 = type { double } -define hidden %0 @__divsc3(float %a, float %b, float %c, float %d) nounwind readnone { +define hidden %0 @__divsc3(float %a, float %b, float %c, float %d) nounwind readnone !dbg !1 { entry: tail call void @llvm.dbg.value(metadata float %a, i64 0, metadata !0, metadata !DIExpression()), !dbg !DILocation(scope: !1) tail call void @llvm.dbg.value(metadata float %b, i64 0, metadata !11, metadata !DIExpression()), !dbg !DILocation(scope: !1) @@ -199,10 +199,10 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon !llvm.dbg.cu = !{!3} !llvm.module.flags = !{!48} -!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 1921, arg: 0, scope: !1, file: !2, type: !9) -!1 = !DISubprogram(name: "__divsc3", linkageName: "__divsc3", line: 1922, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 1922, file: !45, scope: !2, type: !4, function: %0 (float, float, float, float)* @__divsc3, variables: !43) +!0 = !DILocalVariable(name: "a", line: 1921, arg: 1, scope: !1, file: !2, type: !9) +!1 = distinct !DISubprogram(name: "__divsc3", linkageName: "__divsc3", line: 1922, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 1922, file: !45, scope: !2, type: !4, variables: !43) !2 = !DIFile(filename: "libgcc2.c", directory: "/Users/yash/clean/LG.D/gcc/../../llvmgcc/gcc") -!3 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 1, file: !45, enums: !47, retainedTypes: !47, subprograms: !44, imports: null) +!3 = distinct !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 1, file: !45, enums: !47, retainedTypes: !47, subprograms: !44, imports: null) !4 = !DISubroutineType(types: !5) !5 = !{!6, !9, !9, !9, !9} !6 = !DIDerivedType(tag: DW_TAG_typedef, name: "SCtype", line: 170, file: !46, scope: !7, baseType: !8) @@ -210,14 +210,14 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon !8 = !DIBasicType(tag: DW_TAG_base_type, name: "complex float", size: 64, align: 32, encoding: DW_ATE_complex_float) !9 = !DIDerivedType(tag: DW_TAG_typedef, name: "SFtype", line: 167, file: !46, scope: !7, baseType: !10) !10 = !DIBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float) -!11 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 1921, arg: 0, scope: !1, file: !2, type: !9) -!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 1921, arg: 0, scope: !1, file: !2, type: !9) -!13 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "d", line: 1921, arg: 0, scope: !1, file: !2, type: !9) -!14 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "denom", line: 1923, scope: !15, file: !2, type: !9) +!11 = !DILocalVariable(name: "b", line: 1921, arg: 2, scope: !1, file: !2, type: !9) +!12 = !DILocalVariable(name: "c", line: 1921, arg: 3, scope: !1, file: !2, type: !9) +!13 = !DILocalVariable(name: "d", line: 1921, arg: 4, scope: !1, file: !2, type: !9) +!14 = !DILocalVariable(name: "denom", line: 1923, scope: !15, file: !2, type: !9) !15 = distinct !DILexicalBlock(line: 1922, column: 0, file: !45, scope: !1) -!16 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "ratio", line: 1923, scope: !15, file: !2, type: !9) -!17 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 1923, scope: !15, file: !2, type: !9) -!18 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "y", line: 1923, scope: !15, file: !2, type: !9) +!16 = !DILocalVariable(name: "ratio", line: 1923, scope: !15, file: !2, type: !9) +!17 = !DILocalVariable(name: "x", line: 1923, scope: !15, file: !2, type: !9) +!18 = !DILocalVariable(name: "y", line: 1923, scope: !15, file: !2, type: !9) !19 = !DILocation(line: 1929, scope: !15) !20 = !DILocation(line: 1931, scope: !15) !21 = !DILocation(line: 1932, scope: !15) diff --git a/test/CodeGen/X86/2010-05-26-DotDebugLoc.ll b/test/CodeGen/X86/2010-05-26-DotDebugLoc.ll index 3670c556aa79..a34e7bd9fe43 100644 --- a/test/CodeGen/X86/2010-05-26-DotDebugLoc.ll +++ b/test/CodeGen/X86/2010-05-26-DotDebugLoc.ll @@ -7,7 +7,7 @@ target triple = "x86_64-apple-darwin10" @llvm.used = appending global [1 x i8*] [i8* bitcast (i8* (%struct.a*)* @bar to i8*)], section "llvm.metadata" ; <[1 x i8*]*> [#uses=0] -define i8* @bar(%struct.a* %myvar) nounwind optsize noinline ssp { +define i8* @bar(%struct.a* %myvar) nounwind optsize noinline ssp !dbg !9 { entry: tail call void @llvm.dbg.value(metadata %struct.a* %myvar, i64 0, metadata !8, metadata !DIExpression()), !dbg !DILocation(scope: !9) %0 = getelementptr inbounds %struct.a, %struct.a* %myvar, i64 0, i32 0, !dbg !28 ; [#uses=1] @@ -26,14 +26,14 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon !0 = !DIGlobalVariable(name: "ret", line: 7, isLocal: false, isDefinition: true, scope: !1, file: !1, type: !3) !1 = !DIFile(filename: "foo.c", directory: "/tmp/") -!2 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 1, file: !36, enums: !37, retainedTypes: !37, subprograms: !32, globals: !31, imports: !37) +!2 = distinct !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 1, file: !36, enums: !37, retainedTypes: !37, subprograms: !32, globals: !31, imports: !37) !3 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) -!4 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 12, arg: 0, scope: !5, file: !1, type: !3) -!5 = !DISubprogram(name: "foo", linkageName: "foo", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 13, file: !36, scope: !1, type: !6, function: void (i32)* @foo, variables: !33) +!4 = !DILocalVariable(name: "x", line: 12, arg: 1, scope: !5, file: !1, type: !3) +!5 = distinct !DISubprogram(name: "foo", linkageName: "foo", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 13, file: !36, scope: !1, type: !6, variables: !33) !6 = !DISubroutineType(types: !7) !7 = !{null, !3} -!8 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "myvar", line: 17, arg: 0, scope: !9, file: !1, type: !13) -!9 = !DISubprogram(name: "bar", linkageName: "bar", line: 17, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 17, file: !36, scope: !1, type: !10, function: i8* (%struct.a*)* @bar, variables: !34) +!8 = !DILocalVariable(name: "myvar", line: 17, arg: 1, scope: !9, file: !1, type: !13) +!9 = distinct !DISubprogram(name: "bar", linkageName: "bar", line: 17, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 17, file: !36, scope: !1, type: !10, variables: !34) !10 = !DISubroutineType(types: !11) !11 = !{!12, !13} !12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !36, scope: !1, baseType: null) @@ -42,15 +42,15 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon !15 = !{!16, !17} !16 = !DIDerivedType(tag: DW_TAG_member, name: "c", line: 3, size: 32, align: 32, file: !36, scope: !14, baseType: !3) !17 = !DIDerivedType(tag: DW_TAG_member, name: "d", line: 4, size: 64, align: 64, offset: 64, file: !36, scope: !14, baseType: !13) -!18 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 22, arg: 0, scope: !19, file: !1, type: !3) -!19 = !DISubprogram(name: "main", linkageName: "main", line: 22, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 22, file: !36, scope: !1, type: !20, variables: !35) +!18 = !DILocalVariable(name: "argc", line: 22, arg: 1, scope: !19, file: !1, type: !3) +!19 = distinct !DISubprogram(name: "main", linkageName: "main", line: 22, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 22, file: !36, scope: !1, type: !20, variables: !35) !20 = !DISubroutineType(types: !21) !21 = !{!3, !3, !22} !22 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !36, scope: !1, baseType: !23) !23 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !36, scope: !1, baseType: !24) !24 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char) -!25 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 22, arg: 0, scope: !19, file: !1, type: !22) -!26 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "e", line: 23, scope: !27, file: !1, type: !14) +!25 = !DILocalVariable(name: "argv", line: 22, arg: 2, scope: !19, file: !1, type: !22) +!26 = !DILocalVariable(name: "e", line: 23, scope: !27, file: !1, type: !14) !27 = distinct !DILexicalBlock(line: 22, column: 0, file: !36, scope: !19) !28 = !DILocation(line: 18, scope: !29) !29 = distinct !DILexicalBlock(line: 17, column: 0, file: !36, scope: !9) diff --git a/test/CodeGen/X86/2010-05-28-Crash.ll b/test/CodeGen/X86/2010-05-28-Crash.ll index c5201614fdd1..7967d45c2ee8 100644 --- a/test/CodeGen/X86/2010-05-28-Crash.ll +++ b/test/CodeGen/X86/2010-05-28-Crash.ll @@ -2,7 +2,7 @@ ; RUN: llc -mtriple=x86_64-apple-darwin -regalloc=basic < %s | FileCheck %s ; Test to check separate label for inlined function argument. -define i32 @foo(i32 %y) nounwind optsize ssp { +define i32 @foo(i32 %y) nounwind optsize ssp !dbg !1 { entry: tail call void @llvm.dbg.value(metadata i32 %y, i64 0, metadata !0, metadata !DIExpression()), !dbg !DILocation(scope: !1) %0 = tail call i32 (...) @zoo(i32 %y) nounwind, !dbg !9 ; [#uses=1] @@ -13,10 +13,10 @@ declare i32 @zoo(...) declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnone -define i32 @bar(i32 %x) nounwind optsize ssp { +define i32 @bar(i32 %x) nounwind optsize ssp !dbg !8 { entry: tail call void @llvm.dbg.value(metadata i32 %x, i64 0, metadata !7, metadata !DIExpression()), !dbg !DILocation(scope: !8) - tail call void @llvm.dbg.value(metadata i32 1, i64 0, metadata !0, metadata !DIExpression()) nounwind, !dbg !DILocation(scope: !1) + tail call void @llvm.dbg.value(metadata i32 1, i64 0, metadata !0, metadata !DIExpression()) nounwind, !dbg !DILocation(scope: !1, inlinedAt: !DILocation(scope: !8)) %0 = tail call i32 (...) @zoo(i32 1) nounwind, !dbg !12 ; [#uses=1] %1 = add nsw i32 %0, %x, !dbg !13 ; [#uses=1] ret i32 %1, !dbg !13 @@ -25,15 +25,15 @@ entry: !llvm.dbg.cu = !{!3} !llvm.module.flags = !{!20} -!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "y", line: 2, arg: 0, scope: !1, file: !2, type: !6) -!1 = !DISubprogram(name: "foo", linkageName: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 2, file: !18, scope: !2, type: !4, function: i32 (i32)* @foo, variables: !15) +!0 = !DILocalVariable(name: "y", line: 2, arg: 1, scope: !1, file: !2, type: !6) +!1 = distinct !DISubprogram(name: "foo", linkageName: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 2, file: !18, scope: !2, type: !4, variables: !15) !2 = !DIFile(filename: "f.c", directory: "/tmp") -!3 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 1, file: !18, enums: !19, retainedTypes: !19, subprograms: !17, imports: null) +!3 = distinct !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: 1, file: !18, enums: !19, retainedTypes: !19, subprograms: !17, imports: null) !4 = !DISubroutineType(types: !5) !5 = !{!6, !6} !6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) -!7 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 6, arg: 0, scope: !8, file: !2, type: !6) -!8 = !DISubprogram(name: "bar", linkageName: "bar", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 6, file: !18, scope: !2, type: !4, function: i32 (i32)* @bar, variables: !16) +!7 = !DILocalVariable(name: "x", line: 6, arg: 1, scope: !8, file: !2, type: !6) +!8 = distinct !DISubprogram(name: "bar", linkageName: "bar", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 6, file: !18, scope: !2, type: !4, variables: !16) !9 = !DILocation(line: 3, scope: !10) !10 = distinct !DILexicalBlock(line: 2, column: 0, file: !18, scope: !1) !11 = !{i32 1} @@ -46,7 +46,7 @@ entry: !18 = !DIFile(filename: "f.c", directory: "/tmp") !19 = !{} -;CHECK: DEBUG_VALUE: bar:x <- E +;CHECK: DEBUG_VALUE: bar:x <- %E ;CHECK: Ltmp ;CHECK: DEBUG_VALUE: foo:y <- 1{{$}} !20 = !{i32 1, !"Debug Info Version", i32 3} diff --git a/test/CodeGen/X86/2010-06-01-DeadArg-DbgInfo.ll b/test/CodeGen/X86/2010-06-01-DeadArg-DbgInfo.ll index 757c92808e11..1be800cdfcf0 100644 --- a/test/CodeGen/X86/2010-06-01-DeadArg-DbgInfo.ll +++ b/test/CodeGen/X86/2010-06-01-DeadArg-DbgInfo.ll @@ -7,8 +7,8 @@ target triple = "x86_64-apple-darwin10.2" @llvm.used = appending global [1 x i8*] [i8* bitcast (i32 (%struct.foo*, i32)* @_ZN3foo3bazEi to i8*)], section "llvm.metadata" ; <[1 x i8*]*> [#uses=0] -define i32 @_ZN3foo3bazEi(%struct.foo* nocapture %this, i32 %x) nounwind readnone optsize noinline ssp align 2 { -;CHECK: DEBUG_VALUE: baz:this <- RDI{{$}} +define i32 @_ZN3foo3bazEi(%struct.foo* nocapture %this, i32 %x) nounwind readnone optsize noinline ssp align 2 !dbg !8 { +;CHECK: DEBUG_VALUE: baz:this <- %RDI{{$}} entry: tail call void @llvm.dbg.value(metadata %struct.foo* %this, i64 0, metadata !15, metadata !DIExpression()), !dbg !DILocation(scope: !8) tail call void @llvm.dbg.value(metadata i32 %x, i64 0, metadata !16, metadata !DIExpression()), !dbg !DILocation(scope: !8) @@ -23,35 +23,35 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon !llvm.module.flags = !{!34} !llvm.dbg.lv = !{!0, !14, !15, !16, !17, !24, !25, !28} -!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 11, arg: 0, scope: !1, file: !3, type: !12) -!1 = !DISubprogram(name: "bar", linkageName: "_ZN3foo3barEi", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 11, file: !31, scope: !2, type: !9, function: i32 (%struct.foo*, i32)* null) +!0 = !DILocalVariable(name: "this", line: 11, arg: 1, scope: !1, file: !3, type: !12) +!1 = distinct !DISubprogram(name: "bar", linkageName: "_ZN3foo3barEi", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 11, file: !31, scope: !2, type: !9) !2 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 3, size: 32, align: 32, file: !31, scope: !3, elements: !5) !3 = !DIFile(filename: "foo.cp", directory: "/tmp/") -!4 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 LLVM build", isOptimized: true, emissionKind: 0, file: !31, enums: !32, retainedTypes: !32, subprograms: !33) +!4 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 LLVM build", isOptimized: true, emissionKind: 0, file: !31, enums: !32, retainedTypes: !32, subprograms: !33) !5 = !{!6, !1, !8} !6 = !DIDerivedType(tag: DW_TAG_member, name: "y", line: 8, size: 32, align: 32, file: !31, scope: !2, baseType: !7) !7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) -!8 = !DISubprogram(name: "baz", linkageName: "_ZN3foo3bazEi", line: 15, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 15, file: !31, scope: !2, type: !9, function: i32 (%struct.foo*, i32)* @_ZN3foo3bazEi) +!8 = distinct !DISubprogram(name: "baz", linkageName: "_ZN3foo3bazEi", line: 15, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 15, file: !31, scope: !2, type: !9) !9 = !DISubroutineType(types: !10) !10 = !{!7, !11, !7} !11 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !31, scope: !3, baseType: !2) !12 = !DIDerivedType(tag: DW_TAG_const_type, size: 64, align: 64, flags: DIFlagArtificial, file: !31, scope: !3, baseType: !13) !13 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !31, scope: !3, baseType: !2) -!14 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 11, arg: 0, scope: !1, file: !3, type: !7) -!15 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 15, arg: 0, scope: !8, file: !3, type: !12) -!16 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 15, arg: 0, scope: !8, file: !3, type: !7) -!17 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argc", line: 19, arg: 0, scope: !18, file: !3, type: !7) -!18 = !DISubprogram(name: "main", linkageName: "main", line: 19, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 19, file: !31, scope: !3, type: !19) +!14 = !DILocalVariable(name: "x", line: 11, arg: 2, scope: !1, file: !3, type: !7) +!15 = !DILocalVariable(name: "this", line: 15, arg: 1, scope: !8, file: !3, type: !12) +!16 = !DILocalVariable(name: "x", line: 15, arg: 2, scope: !8, file: !3, type: !7) +!17 = !DILocalVariable(name: "argc", line: 19, arg: 1, scope: !18, file: !3, type: !7) +!18 = distinct !DISubprogram(name: "main", linkageName: "main", line: 19, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, scopeLine: 19, file: !31, scope: !3, type: !19) !19 = !DISubroutineType(types: !20) !20 = !{!7, !7, !21} !21 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !31, scope: !3, baseType: !22) !22 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !31, scope: !3, baseType: !23) !23 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char) -!24 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "argv", line: 19, arg: 0, scope: !18, file: !3, type: !21) -!25 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 20, scope: !26, file: !3, type: !2) +!24 = !DILocalVariable(name: "argv", line: 19, arg: 2, scope: !18, file: !3, type: !21) +!25 = !DILocalVariable(name: "a", line: 20, scope: !26, file: !3, type: !2) !26 = distinct !DILexicalBlock(line: 19, column: 0, file: !31, scope: !27) !27 = distinct !DILexicalBlock(line: 19, column: 0, file: !31, scope: !18) -!28 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 21, scope: !26, file: !3, type: !7) +!28 = !DILocalVariable(name: "b", line: 21, scope: !26, file: !3, type: !7) !29 = !DILocation(line: 16, scope: !30) !30 = distinct !DILexicalBlock(line: 15, column: 0, file: !31, scope: !8) !31 = !DIFile(filename: "foo.cp", directory: "/tmp/") diff --git a/test/CodeGen/X86/2010-07-06-DbgCrash.ll b/test/CodeGen/X86/2010-07-06-DbgCrash.ll index 3ce36eec400a..5e565a1a667f 100644 --- a/test/CodeGen/X86/2010-07-06-DbgCrash.ll +++ b/test/CodeGen/X86/2010-07-06-DbgCrash.ll @@ -1,18 +1,19 @@ ; RUN: llc -O0 -relocation-model pic < %s -o /dev/null +; REQUIRES: default_triple ; PR7545 @.str = private constant [4 x i8] c"one\00", align 1 ; <[4 x i8]*> [#uses=1] @.str1 = private constant [4 x i8] c"two\00", align 1 ; <[5 x i8]*> [#uses=1] @C.9.2167 = internal constant [2 x i8*] [i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str1, i64 0, i64 0)] !38 = !DIFile(filename: "pbmsrch.c", directory: "/Users/grawp/LLVM/test-suite/MultiSource/Benchmarks/MiBench/office-stringsearch") -!39 = !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build 9999)", isOptimized: true, emissionKind: 0, file: !109, enums: !108, retainedTypes: !108) +!39 = distinct !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build 9999)", isOptimized: true, emissionKind: 0, file: !109, enums: !108, retainedTypes: !108) !46 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !109, baseType: !47) !47 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char) -!97 = !DISubprogram(name: "main", linkageName: "main", line: 73, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !39, type: !98) +!97 = distinct !DISubprogram(name: "main", linkageName: "main", line: 73, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scope: !39, type: !98) !98 = !DISubroutineType(types: !99) !99 = !{!100} !100 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !101 = !{[2 x i8*]* @C.9.2167} -!102 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "find_strings", line: 75, scope: !103, file: !38, type: !104) +!102 = !DILocalVariable(name: "find_strings", line: 75, scope: !103, file: !38, type: !104) !103 = distinct !DILexicalBlock(line: 73, column: 0, file: null, scope: !97) !104 = !DICompositeType(tag: DW_TAG_array_type, size: 85312, align: 64, file: !109, baseType: !46, elements: !105) !105 = !{!106} diff --git a/test/CodeGen/X86/2010-08-04-StackVariable.ll b/test/CodeGen/X86/2010-08-04-StackVariable.ll index 6129e78fd348..d305d678c596 100644 --- a/test/CodeGen/X86/2010-08-04-StackVariable.ll +++ b/test/CodeGen/X86/2010-08-04-StackVariable.ll @@ -3,7 +3,7 @@ %struct.SVal = type { i8*, i32 } -define i32 @_Z3fooi4SVal(i32 %i, %struct.SVal* noalias %location) nounwind ssp { +define i32 @_Z3fooi4SVal(i32 %i, %struct.SVal* noalias %location) nounwind ssp !dbg !17 { entry: %"alloca point" = bitcast i32 0 to i32 ; [#uses=0] call void @llvm.dbg.value(metadata i32 %i, i64 0, metadata !23, metadata !DIExpression()), !dbg !24 @@ -31,7 +31,7 @@ return: ; preds = %bb2 ret i32 %.0, !dbg !29 } -define linkonce_odr void @_ZN4SValC1Ev(%struct.SVal* %this) nounwind ssp align 2 { +define linkonce_odr void @_ZN4SValC1Ev(%struct.SVal* %this) nounwind ssp align 2 !dbg !16 { entry: %"alloca point" = bitcast i32 0 to i32 ; [#uses=0] call void @llvm.dbg.value(metadata %struct.SVal* %this, i64 0, metadata !31, metadata !DIExpression()), !dbg !34 @@ -47,7 +47,7 @@ return: ; preds = %entry declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone -define i32 @main() nounwind ssp { +define i32 @main() nounwind ssp !dbg !20 { entry: %0 = alloca %struct.SVal ; <%struct.SVal*> [#uses=3] %v = alloca %struct.SVal ; <%struct.SVal*> [#uses=4] @@ -81,7 +81,7 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon !0 = !DISubprogram(name: "SVal", line: 11, isLocal: false, isDefinition: false, virtualIndex: 6, isOptimized: false, scopeLine: 11, file: !47, scope: !1, type: !14) !1 = !DICompositeType(tag: DW_TAG_structure_type, name: "SVal", line: 1, size: 128, align: 64, file: !47, scope: !2, elements: !4) !2 = !DIFile(filename: "small.cc", directory: "/Users/manav/R8248330") -!3 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 1, file: !47, enums: !48, retainedTypes: !48, subprograms: !46, imports: null) +!3 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: 1, file: !47, enums: !48, retainedTypes: !48, subprograms: !46, imports: null) !4 = !{!5, !7, !0, !9} !5 = !DIDerivedType(tag: DW_TAG_member, name: "Data", line: 7, size: 64, align: 64, file: !47, scope: !1, baseType: !6) !6 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !47, scope: !2, baseType: null) @@ -94,35 +94,35 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon !13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !14 = !DISubroutineType(types: !15) !15 = !{null, !12} -!16 = !DISubprogram(name: "SVal", linkageName: "_ZN4SValC1Ev", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 11, file: !47, scope: !1, type: !14, function: void (%struct.SVal*)* @_ZN4SValC1Ev) -!17 = !DISubprogram(name: "foo", linkageName: "_Z3fooi4SVal", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 16, file: !47, scope: !2, type: !18, function: i32 (i32, %struct.SVal*)* @_Z3fooi4SVal) +!16 = distinct !DISubprogram(name: "SVal", linkageName: "_ZN4SValC1Ev", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 11, file: !47, scope: !1, type: !14) +!17 = distinct !DISubprogram(name: "foo", linkageName: "_Z3fooi4SVal", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 16, file: !47, scope: !2, type: !18) !18 = !DISubroutineType(types: !19) !19 = !{!13, !13, !1} -!20 = !DISubprogram(name: "main", linkageName: "main", line: 23, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 23, file: !47, scope: !2, type: !21, function: i32 ()* @main) +!20 = distinct !DISubprogram(name: "main", linkageName: "main", line: 23, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 23, file: !47, scope: !2, type: !21) !21 = !DISubroutineType(types: !22) !22 = !{!13} -!23 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 16, arg: 0, scope: !17, file: !2, type: !13) +!23 = !DILocalVariable(name: "i", line: 16, arg: 1, scope: !17, file: !2, type: !13) !24 = !DILocation(line: 16, scope: !17) -!25 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "location", line: 16, arg: 0, scope: !17, file: !2, type: !26) +!25 = !DILocalVariable(name: "location", line: 16, arg: 2, scope: !17, file: !2, type: !26) !26 = !DIDerivedType(tag: DW_TAG_reference_type, name: "SVal", size: 64, align: 64, file: !47, scope: !2, baseType: !1) !27 = !DILocation(line: 17, scope: !28) !28 = distinct !DILexicalBlock(line: 16, column: 0, file: !47, scope: !17) !29 = !DILocation(line: 18, scope: !28) !30 = !DILocation(line: 20, scope: !28) -!31 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 11, arg: 0, scope: !16, file: !2, type: !32) +!31 = !DILocalVariable(name: "this", line: 11, arg: 1, scope: !16, file: !2, type: !32) !32 = !DIDerivedType(tag: DW_TAG_const_type, size: 64, align: 64, flags: DIFlagArtificial, file: !47, scope: !2, baseType: !33) !33 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !47, scope: !2, baseType: !1) !34 = !DILocation(line: 11, scope: !16) !35 = !DILocation(line: 11, scope: !36) !36 = distinct !DILexicalBlock(line: 11, column: 0, file: !47, scope: !37) !37 = distinct !DILexicalBlock(line: 11, column: 0, file: !47, scope: !16) -!38 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "v", line: 24, scope: !39, file: !2, type: !1) +!38 = !DILocalVariable(name: "v", line: 24, scope: !39, file: !2, type: !1) !39 = distinct !DILexicalBlock(line: 23, column: 0, file: !47, scope: !40) !40 = distinct !DILexicalBlock(line: 23, column: 0, file: !47, scope: !20) !41 = !DILocation(line: 24, scope: !39) !42 = !DILocation(line: 25, scope: !39) !43 = !DILocation(line: 26, scope: !39) -!44 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "k", line: 26, scope: !39, file: !2, type: !13) +!44 = !DILocalVariable(name: "k", line: 26, scope: !39, file: !2, type: !13) !45 = !DILocation(line: 27, scope: !39) !47 = !DIFile(filename: "small.cc", directory: "/Users/manav/R8248330") !48 = !{} diff --git a/test/CodeGen/X86/2010-09-16-EmptyFilename.ll b/test/CodeGen/X86/2010-09-16-EmptyFilename.ll index d94bd1c79f91..4303ca991a86 100644 --- a/test/CodeGen/X86/2010-09-16-EmptyFilename.ll +++ b/test/CodeGen/X86/2010-09-16-EmptyFilename.ll @@ -2,12 +2,12 @@ ; Radar 8286101 ; CHECK: .file {{[0-9]+}} "" -define i32 @foo() nounwind ssp { +define i32 @foo() nounwind ssp !dbg !0 { entry: ret i32 42, !dbg !8 } -define i32 @bar() nounwind ssp { +define i32 @bar() nounwind ssp !dbg !6 { entry: ret i32 21, !dbg !10 } @@ -15,13 +15,13 @@ entry: !llvm.dbg.cu = !{!2} !llvm.module.flags = !{!17} -!0 = !DISubprogram(name: "foo", linkageName: "foo", line: 53, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !14, scope: !1, type: !3, function: i32 ()* @foo) +!0 = distinct !DISubprogram(name: "foo", linkageName: "foo", line: 53, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !14, scope: !1, type: !3) !1 = !DIFile(filename: "", directory: "/private/tmp") -!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 114084)", isOptimized: false, emissionKind: 0, file: !15, enums: !16, retainedTypes: !16, subprograms: !13) +!2 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 114084)", isOptimized: false, emissionKind: 0, file: !15, enums: !16, retainedTypes: !16, subprograms: !13) !3 = !DISubroutineType(types: !4) !4 = !{!5} !5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) -!6 = !DISubprogram(name: "bar", linkageName: "bar", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !15, scope: !7, type: !3, function: i32 ()* @bar) +!6 = distinct !DISubprogram(name: "bar", linkageName: "bar", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, file: !15, scope: !7, type: !3) !7 = !DIFile(filename: "bug.c", directory: "/private/tmp") !8 = !DILocation(line: 53, column: 13, scope: !9) !9 = distinct !DILexicalBlock(line: 53, column: 11, file: !14, scope: !0) diff --git a/test/CodeGen/X86/2010-11-02-DbgParameter.ll b/test/CodeGen/X86/2010-11-02-DbgParameter.ll index 124cc9a430e8..b091003585c2 100644 --- a/test/CodeGen/X86/2010-11-02-DbgParameter.ll +++ b/test/CodeGen/X86/2010-11-02-DbgParameter.ll @@ -6,7 +6,7 @@ target triple = "i386-apple-darwin11.0.0" %struct.bar = type { i32, i32 } -define i32 @foo(%struct.bar* nocapture %i) nounwind readnone optsize noinline ssp { +define i32 @foo(%struct.bar* nocapture %i) nounwind readnone optsize noinline ssp !dbg !0 { ; CHECK: TAG_formal_parameter entry: tail call void @llvm.dbg.value(metadata %struct.bar* %i, i64 0, metadata !6, metadata !DIExpression()), !dbg !12 @@ -18,13 +18,13 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon !llvm.dbg.cu = !{!2} !llvm.module.flags = !{!19} -!0 = !DISubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !17, scope: !1, type: !3, function: i32 (%struct.bar*)* @foo, variables: !16) +!0 = distinct !DISubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 3, file: !17, scope: !1, type: !3, variables: !16) !1 = !DIFile(filename: "one.c", directory: "/private/tmp") -!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 117922)", isOptimized: true, emissionKind: 0, file: !17, enums: !18, retainedTypes: !18, subprograms: !15, imports: null) +!2 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 117922)", isOptimized: true, emissionKind: 0, file: !17, enums: !18, retainedTypes: !18, subprograms: !15, imports: null) !3 = !DISubroutineType(types: !4) !4 = !{!5} !5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) -!6 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 3, arg: 0, scope: !0, file: !1, type: !7) +!6 = !DILocalVariable(name: "i", line: 3, arg: 1, scope: !0, file: !1, type: !7) !7 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, file: !17, scope: !1, baseType: !8) !8 = !DICompositeType(tag: DW_TAG_structure_type, name: "bar", line: 2, size: 64, align: 32, file: !17, scope: !1, elements: !9) !9 = !{!10, !11} diff --git a/test/CodeGen/X86/2011-01-24-DbgValue-Before-Use.ll b/test/CodeGen/X86/2011-01-24-DbgValue-Before-Use.ll index 0ded66fa3bf9..661ec94fee4e 100644 --- a/test/CodeGen/X86/2011-01-24-DbgValue-Before-Use.ll +++ b/test/CodeGen/X86/2011-01-24-DbgValue-Before-Use.ll @@ -20,7 +20,7 @@ target triple = "x86_64-apple-darwin10.0.0" @.str1 = private unnamed_addr constant [14 x i8] c"m=%u, z_s=%d\0A\00" @str = internal constant [21 x i8] c"Failing test vector:\00" -define i64 @gcd(i64 %a, i64 %b) nounwind readnone optsize noinline ssp { +define i64 @gcd(i64 %a, i64 %b) nounwind readnone optsize noinline ssp !dbg !0 { entry: tail call void @llvm.dbg.value(metadata i64 %a, i64 0, metadata !10, metadata !DIExpression()), !dbg !18 tail call void @llvm.dbg.value(metadata i64 %b, i64 0, metadata !11, metadata !DIExpression()), !dbg !19 @@ -38,7 +38,7 @@ if.then: ; preds = %while.body ret i64 %b.addr.0, !dbg !23 } -define i32 @main() nounwind optsize ssp { +define i32 @main() nounwind optsize ssp !dbg !6 { entry: %call = tail call i32 @rand() nounwind optsize, !dbg !24 tail call void @llvm.dbg.value(metadata i32 %call, i64 0, metadata !14, metadata !DIExpression()), !dbg !24 @@ -78,24 +78,24 @@ declare i32 @puts(i8* nocapture) nounwind !llvm.dbg.cu = !{!2} !llvm.module.flags = !{!33} -!0 = !DISubprogram(name: "gcd", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !31, scope: !1, type: !3, function: i64 (i64, i64)* @gcd, variables: !29) +!0 = distinct !DISubprogram(name: "gcd", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !31, scope: !1, type: !3, variables: !29) !1 = !DIFile(filename: "rem_small.c", directory: "/private/tmp") -!2 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 124117)", isOptimized: true, emissionKind: 1, file: !31, enums: !32, retainedTypes: !32, subprograms: !28, imports: null) +!2 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 124117)", isOptimized: true, emissionKind: 1, file: !31, enums: !32, retainedTypes: !32, subprograms: !28, imports: null) !3 = !DISubroutineType(types: !4) !4 = !{!5} !5 = !DIBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed) -!6 = !DISubprogram(name: "main", line: 25, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !31, scope: !1, type: !7, function: i32 ()* @main, variables: !30) +!6 = distinct !DISubprogram(name: "main", line: 25, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, file: !31, scope: !1, type: !7, variables: !30) !7 = !DISubroutineType(types: !8) !8 = !{!9} !9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) -!10 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "a", line: 5, arg: 0, scope: !0, file: !1, type: !5) -!11 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "b", line: 5, arg: 0, scope: !0, file: !1, type: !5) -!12 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 6, scope: !13, file: !1, type: !5) +!10 = !DILocalVariable(name: "a", line: 5, arg: 1, scope: !0, file: !1, type: !5) +!11 = !DILocalVariable(name: "b", line: 5, arg: 2, scope: !0, file: !1, type: !5) +!12 = !DILocalVariable(name: "c", line: 6, scope: !13, file: !1, type: !5) !13 = distinct !DILexicalBlock(line: 5, column: 52, file: !31, scope: !0) -!14 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "m", line: 26, scope: !15, file: !1, type: !16) +!14 = !DILocalVariable(name: "m", line: 26, scope: !15, file: !1, type: !16) !15 = distinct !DILexicalBlock(line: 25, column: 12, file: !31, scope: !6) !16 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned) -!17 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "z_s", line: 27, scope: !15, file: !1, type: !9) +!17 = !DILocalVariable(name: "z_s", line: 27, scope: !15, file: !1, type: !9) !18 = !DILocation(line: 5, column: 41, scope: !0) !19 = !DILocation(line: 5, column: 49, scope: !0) !20 = !DILocation(line: 7, column: 5, scope: !13) diff --git a/test/CodeGen/X86/2011-10-21-widen-cmp.ll b/test/CodeGen/X86/2011-10-21-widen-cmp.ll index 2fe645b07815..cb4648c382f7 100644 --- a/test/CodeGen/X86/2011-10-21-widen-cmp.ll +++ b/test/CodeGen/X86/2011-10-21-widen-cmp.ll @@ -1,15 +1,23 @@ -; RUN: llc < %s -march=x86-64 -mcpu=corei7 | FileCheck %s - -target triple = "x86_64-unknown-linux-gnu" +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 | FileCheck %s ; Check that a <4 x float> compare is generated and that we are ; not stuck in an endless loop. -; CHECK: cmp_2_floats -; CHECK: cmpordps -; CHECK: ret - define void @cmp_2_floats() { +; CHECK-LABEL: cmp_2_floats: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: cmpordps %xmm0, %xmm0 +; CHECK-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero +; CHECK-NEXT: psllq $32, %xmm0 +; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] +; CHECK-NEXT: psrad $31, %xmm0 +; CHECK-NEXT: pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7] +; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; CHECK-NEXT: pslld $31, %xmm0 +; CHECK-NEXT: blendvps %xmm0, %xmm0 +; CHECK-NEXT: movlps %xmm0, (%rax) +; CHECK-NEXT: retq entry: %0 = fcmp oeq <2 x float> undef, undef %1 = select <2 x i1> %0, <2 x float> undef, <2 x float> undef @@ -17,11 +25,13 @@ entry: ret void } -; CHECK: cmp_2_doubles -; CHECK: cmpordpd -; CHECK: blendvpd -; CHECK: ret define void @cmp_2_doubles() { +; CHECK-LABEL: cmp_2_doubles: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: cmpordpd %xmm0, %xmm0 +; CHECK-NEXT: blendvpd %xmm0, %xmm0 +; CHECK-NEXT: movapd %xmm0, (%rax) +; CHECK-NEXT: retq entry: %0 = fcmp oeq <2 x double> undef, undef %1 = select <2 x i1> %0, <2 x double> undef, <2 x double> undef @@ -29,11 +39,11 @@ entry: ret void } -; CHECK: mp_11193 -; CHECK: psraw $15 -; CHECK: ret -define void @mp_11193(<8 x float> * nocapture %aFOO, <8 x float>* nocapture %RET) -nounwind { +define void @mp_11193(<8 x float> * nocapture %aFOO, <8 x float>* nocapture %RET) nounwind { +; CHECK-LABEL: mp_11193: +; CHECK: # BB#0: # %allocas +; CHECK-NEXT: movl $-1082130432, (%rsi) # imm = 0xFFFFFFFFBF800000 +; CHECK-NEXT: retq allocas: %bincmp = fcmp olt <8 x float> , %t = extractelement <8 x i1> %bincmp, i32 0 diff --git a/test/CodeGen/X86/2011-12-06-AVXVectorExtractCombine.ll b/test/CodeGen/X86/2011-12-06-AVXVectorExtractCombine.ll index 2a1a5c9fb3ea..e6ba7551421d 100644 --- a/test/CodeGen/X86/2011-12-06-AVXVectorExtractCombine.ll +++ b/test/CodeGen/X86/2011-12-06-AVXVectorExtractCombine.ll @@ -4,13 +4,14 @@ define void @test(<4 x i32>* nocapture %p) nounwind { ; CHECK-LABEL: test: ; CHECK: vpxor %xmm0, %xmm0, %xmm0 - ; CHECK-NEXT: vpmaxsd {{.*}}, %xmm0, %xmm0 - ; CHECK-NEXT: vmovdqu %xmm0, (%rdi) + ; CHECK-NEXT: vpmaxsd (%rdi), %xmm0, %xmm0 + ; CHECK-NEXT: vmovdqu %xmm0, (%rdi) ; CHECK-NEXT: ret - %a = call <4 x i32> @llvm.x86.sse41.pmaxsd(<4 x i32> , <4 x i32> zeroinitializer) nounwind - %b = shufflevector <4 x i32> %a, <4 x i32> undef, <8 x i32> - %c = shufflevector <8 x i32> %b, <8 x i32> undef, <4 x i32> - store <4 x i32> %c, <4 x i32>* %p, align 1 + %a = load <4 x i32>, <4 x i32>* %p, align 1 + %b = call <4 x i32> @llvm.x86.sse41.pmaxsd(<4 x i32> %a, <4 x i32> zeroinitializer) nounwind + %c = shufflevector <4 x i32> %b, <4 x i32> undef, <8 x i32> + %d = shufflevector <8 x i32> %c, <8 x i32> undef, <4 x i32> + store <4 x i32> %d, <4 x i32>* %p, align 1 ret void } diff --git a/test/CodeGen/X86/2011-20-21-zext-ui2fp.ll b/test/CodeGen/X86/2011-20-21-zext-ui2fp.ll index 78cdfcf0e1f0..539d5547d5f1 100644 --- a/test/CodeGen/X86/2011-20-21-zext-ui2fp.ll +++ b/test/CodeGen/X86/2011-20-21-zext-ui2fp.ll @@ -1,13 +1,17 @@ -; RUN: llc < %s -march=x86-64 -mcpu=corei7 | FileCheck %s -target triple = "x86_64-unknown-linux-gnu" +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 | FileCheck %s ; Check that the booleans are converted using zext and not via sext. ; 0x1 means that we only look at the first bit. -;CHECK: 0x1 -;CHECK-LABEL: ui_to_fp_conv: -;CHECK: ret define void @ui_to_fp_conv(<8 x float> * nocapture %aFOO, <8 x float>* nocapture %RET) nounwind { +; CHECK-LABEL: ui_to_fp_conv: +; CHECK: # BB#0: # %allocas +; CHECK-NEXT: movaps {{.*#+}} xmm0 = [1.000000e+00,1.000000e+00,0.000000e+00,0.000000e+00] +; CHECK-NEXT: xorps %xmm1, %xmm1 +; CHECK-NEXT: movups %xmm1, 16(%rsi) +; CHECK-NEXT: movups %xmm0, (%rsi) +; CHECK-NEXT: retq allocas: %bincmp = fcmp olt <8 x float> , %bool2float = uitofp <8 x i1> %bincmp to <8 x float> diff --git a/test/CodeGen/X86/2012-01-12-extract-sv.ll b/test/CodeGen/X86/2012-01-12-extract-sv.ll index 677c902668bc..92ec107a0079 100644 --- a/test/CodeGen/X86/2012-01-12-extract-sv.ll +++ b/test/CodeGen/X86/2012-01-12-extract-sv.ll @@ -2,20 +2,20 @@ define void @endless_loop() { ; CHECK-LABEL: endless_loop: -; CHECK-NEXT: # BB#0: -; CHECK-NEXT: vmovaps (%eax), %ymm0 -; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm0 -; CHECK-NEXT: vmovsldup %xmm0, %xmm0 # xmm0 = xmm0[0,0,2,2] -; CHECK-NEXT: vmovddup %xmm0, %xmm1 # xmm1 = xmm0[0,0] -; CHECK-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 -; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2 -; CHECK-NEXT: vblendps $128, %ymm1, %ymm2, %ymm1 # ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7] -; CHECK-NEXT: vxorps %ymm2, %ymm2, %ymm2 -; CHECK-NEXT: vblendps $1, %ymm0, %ymm2, %ymm0 # ymm0 = ymm0[0],ymm2[1,2,3,4,5,6,7] -; CHECK-NEXT: vmovaps %ymm0, (%eax) -; CHECK-NEXT: vmovaps %ymm1, (%eax) -; CHECK-NEXT: vzeroupper -; CHECK-NEXT: retl +; CHECK-NEXT: # BB#0: +; CHECK-NEXT: vmovaps (%eax), %ymm0 +; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm0 +; CHECK-NEXT: vmovsldup {{.*#+}} xmm0 = xmm0[0,0,2,2] +; CHECK-NEXT: vmovddup {{.*#+}} xmm1 = xmm0[0,0] +; CHECK-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 +; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2 +; CHECK-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7] +; CHECK-NEXT: vxorps %ymm2, %ymm2, %ymm2 +; CHECK-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3,4,5,6,7] +; CHECK-NEXT: vmovaps %ymm0, (%eax) +; CHECK-NEXT: vmovaps %ymm1, (%eax) +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retl entry: %0 = load <8 x i32>, <8 x i32> addrspace(1)* undef, align 32 %1 = shufflevector <8 x i32> %0, <8 x i32> undef, <16 x i32> diff --git a/test/CodeGen/X86/2012-08-17-legalizer-crash.ll b/test/CodeGen/X86/2012-08-17-legalizer-crash.ll index a19aa52f302f..816577be15e7 100644 --- a/test/CodeGen/X86/2012-08-17-legalizer-crash.ll +++ b/test/CodeGen/X86/2012-08-17-legalizer-crash.ll @@ -26,6 +26,5 @@ if.end: ; preds = %if.then, %entry ret void ; CHECK-LABEL: fn1: -; CHECK: shrq $32, [[REG:%.*]] -; CHECK: sete +; CHECK: jb } diff --git a/test/CodeGen/X86/2012-1-10-buildvector.ll b/test/CodeGen/X86/2012-1-10-buildvector.ll index d1c0266941fd..eb237847e1bc 100644 --- a/test/CodeGen/X86/2012-1-10-buildvector.ll +++ b/test/CodeGen/X86/2012-1-10-buildvector.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mattr=+avx -mtriple=i686-unknown-unknown | FileCheck %s define void @bad_cast() { diff --git a/test/CodeGen/X86/2012-11-30-handlemove-dbg.ll b/test/CodeGen/X86/2012-11-30-handlemove-dbg.ll index a27db95ba127..50b486c6f925 100644 --- a/test/CodeGen/X86/2012-11-30-handlemove-dbg.ll +++ b/test/CodeGen/X86/2012-11-30-handlemove-dbg.ll @@ -14,7 +14,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone -define signext i16 @subdivp(%struct.node.0.27* nocapture %p, double %dsq, double %tolsq, %struct.hgstruct.2.29* nocapture byval align 8 %hg) nounwind uwtable readonly ssp { +define signext i16 @subdivp(%struct.node.0.27* nocapture %p, double %dsq, double %tolsq, %struct.hgstruct.2.29* nocapture byval align 8 %hg) nounwind uwtable readonly ssp !dbg !14 { entry: call void @llvm.dbg.declare(metadata %struct.hgstruct.2.29* %hg, metadata !4, metadata !DIExpression()), !dbg !DILocation(scope: !14) %type = getelementptr inbounds %struct.node.0.27, %struct.node.0.27* %p, i64 0, i32 0 @@ -38,15 +38,15 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!12} -!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 168918) (llvm/trunk 168920)", isOptimized: true, emissionKind: 0, file: !11, enums: !2, retainedTypes: !2, subprograms: !13, globals: !2) +!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 168918) (llvm/trunk 168920)", isOptimized: true, emissionKind: 0, file: !11, enums: !2, retainedTypes: !2, subprograms: !13, globals: !2) !2 = !{} -!4 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "hg", line: 725, arg: 4, scope: !14, file: !5, type: !6) +!4 = !DILocalVariable(name: "hg", line: 725, arg: 4, scope: !14, file: !5, type: !6) !5 = !DIFile(filename: "MultiSource/Benchmarks/Olden/bh/newbh.c", directory: "MultiSource/Benchmarks/Olden/bh") !6 = !DIDerivedType(tag: DW_TAG_typedef, name: "hgstruct", line: 492, file: !11, baseType: !7) !7 = !DICompositeType(tag: DW_TAG_structure_type, line: 487, size: 512, align: 64, file: !11) !11 = !DIFile(filename: "MultiSource/Benchmarks/Olden/bh/newbh.c", directory: "MultiSource/Benchmarks/Olden/bh") !12 = !{i32 1, !"Debug Info Version", i32 3} !13 = !{!14} -!14 = !DISubprogram(name: "subdivp", isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !11, scope: !5, type: !15, function: i16 (%struct.node.0.27*, double, double, %struct.hgstruct.2.29* )* @subdivp) +!14 = distinct !DISubprogram(name: "subdivp", isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !11, scope: !5, type: !15) !15 = !DISubroutineType(types: !16) !16 = !{null} diff --git a/test/CodeGen/X86/2012-11-30-misched-dbg.ll b/test/CodeGen/X86/2012-11-30-misched-dbg.ll index 22227faab942..7ed416e36c22 100644 --- a/test/CodeGen/X86/2012-11-30-misched-dbg.ll +++ b/test/CodeGen/X86/2012-11-30-misched-dbg.ll @@ -14,7 +14,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone -define i32 @AttachGalley(%union.rec** nocapture %suspend_pt) nounwind uwtable ssp { +define i32 @AttachGalley(%union.rec** nocapture %suspend_pt) nounwind uwtable ssp !dbg !21 { entry: %num14075 = alloca [20 x i8], align 16 br label %if.end33 @@ -65,10 +65,10 @@ declare i32 @__sprintf_chk(i8*, i32, i64, i8*, ...) !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!35} -!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 168918) (llvm/trunk 168920)", isOptimized: true, emissionKind: 0, file: !19, enums: !2, retainedTypes: !2, subprograms: !20, globals: !2) +!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.3 (trunk 168918) (llvm/trunk 168920)", isOptimized: true, emissionKind: 0, file: !19, enums: !2, retainedTypes: !2, subprograms: !20, globals: !2) !1 = !{!2} !2 = !{} -!4 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "num1", line: 815, scope: !5, file: !14, type: !15) +!4 = !DILocalVariable(name: "num1", line: 815, scope: !5, file: !14, type: !15) !5 = distinct !DILexicalBlock(line: 815, column: 0, file: !14, scope: !6) !6 = distinct !DILexicalBlock(line: 812, column: 0, file: !14, scope: !7) !7 = distinct !DILexicalBlock(line: 807, column: 0, file: !14, scope: !8) @@ -86,7 +86,7 @@ declare i32 @__sprintf_chk(i8*, i32, i64, i8*, ...) !19 = !DIFile(filename: "MultiSource/Benchmarks/MiBench/consumer-typeset/z19.c", directory: "MultiSource/Benchmarks/MiBench/consumer-typeset") !20 = !{!21} -!21 = !DISubprogram(name: "AttachGalley", isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !19, scope: !14, type: !22, function: i32 (%union.rec**)* @AttachGalley) +!21 = distinct !DISubprogram(name: "AttachGalley", isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !19, scope: !14, type: !22) !22 = !DISubroutineType(types: !23) !23 = !{null} @@ -99,7 +99,7 @@ declare i32 @__sprintf_chk(i8*, i32, i64, i8*, ...) %"class.__gnu_cxx::hash_map" = type { %"class.__gnu_cxx::hashtable" } %"class.__gnu_cxx::hashtable" = type { i64, i64, i64, i64, i64, i64 } -define void @main() uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +define void @main() uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) !dbg !37 { entry: %X = alloca %"class.__gnu_cxx::hash_map", align 8 br i1 undef, label %cond.true, label %cond.end @@ -134,11 +134,11 @@ declare void @_Znwm() !llvm.dbg.cu = !{!30} -!30 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 169129) (llvm/trunk 169135)", isOptimized: true, emissionKind: 0, file: !34, enums: !2, retainedTypes: !2, subprograms: !36) -!31 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "X", line: 29, scope: !37, type: !32) +!30 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 169129) (llvm/trunk 169135)", isOptimized: true, emissionKind: 0, file: !34, enums: !2, retainedTypes: !2, subprograms: !36) +!31 = !DILocalVariable(name: "X", line: 29, scope: !37, type: !32) !32 = !DIDerivedType(tag: DW_TAG_typedef, name: "HM", line: 28, file: !34, baseType: null) !33 = !DIFile(filename: "SingleSource/Benchmarks/Shootout-C++/hash.cpp", directory: "SingleSource/Benchmarks/Shootout-C++") !34 = !DIFile(filename: "SingleSource/Benchmarks/Shootout-C++/hash.cpp", directory: "SingleSource/Benchmarks/Shootout-C++") !35 = !{i32 1, !"Debug Info Version", i32 3} !36 = !{!37} -!37 = !DISubprogram(name: "main", isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !19, scope: !14, type: !22, function: void ()* @main) +!37 = distinct !DISubprogram(name: "main", isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !19, scope: !14, type: !22) diff --git a/test/CodeGen/X86/2012-11-30-regpres-dbg.ll b/test/CodeGen/X86/2012-11-30-regpres-dbg.ll index 77c017eb0e36..3f7a10ae035b 100644 --- a/test/CodeGen/X86/2012-11-30-regpres-dbg.ll +++ b/test/CodeGen/X86/2012-11-30-regpres-dbg.ll @@ -11,7 +11,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone -define void @test() unnamed_addr uwtable ssp align 2 { +define void @test() unnamed_addr uwtable ssp align 2 !dbg !2 { entry: %callback = alloca %struct.btCompoundLeafCallback, align 8 br i1 undef, label %if.end, label %if.then @@ -36,10 +36,10 @@ invoke.cont44: ; preds = %if.end !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!8} -!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 168984) (llvm/trunk 168983)", isOptimized: true, emissionKind: 0, file: !6, subprograms: !1) +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 168984) (llvm/trunk 168983)", isOptimized: true, emissionKind: 0, file: !6, subprograms: !1) !1 = !{!2} -!2 = !DISubprogram(name: "test", isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !6, scope: !5, type: !7, function: void ()* @test) -!3 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "callback", line: 214, scope: !2, type: !4) +!2 = distinct !DISubprogram(name: "test", isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 1, file: !6, scope: !5, type: !7) +!3 = !DILocalVariable(name: "callback", line: 214, scope: !2, type: !4) !4 = !DICompositeType(tag: DW_TAG_structure_type, name: "btCompoundLeafCallback", line: 90, size: 512, align: 64, file: !6) !5 = !DIFile(filename: "MultiSource/Benchmarks/Bullet/btCompoundCollisionAlgorithm.cpp", directory: "MultiSource/Benchmarks/Bullet") !6 = !DIFile(filename: "MultiSource/Benchmarks/Bullet/btCompoundCollisionAlgorithm.cpp", directory: "MultiSource/Benchmarks/Bullet") diff --git a/test/CodeGen/X86/3dnow-intrinsics.ll b/test/CodeGen/X86/3dnow-intrinsics.ll index 0b27bf2d1853..fe8b95ec4655 100644 --- a/test/CodeGen/X86/3dnow-intrinsics.ll +++ b/test/CodeGen/X86/3dnow-intrinsics.ll @@ -277,7 +277,7 @@ entry: declare x86_mmx @llvm.x86.3dnowa.pi2fw(x86_mmx) nounwind readnone define <2 x float> @test_pswapdsf(<2 x float> %a) nounwind readnone { -; CHECK: pswapd +; CHECK: pswapd {{.*#+}} mm0 = mem[1,0] entry: %0 = bitcast <2 x float> %a to x86_mmx %1 = tail call x86_mmx @llvm.x86.3dnowa.pswapd(x86_mmx %0) @@ -286,7 +286,7 @@ entry: } define <2 x i32> @test_pswapdsi(<2 x i32> %a) nounwind readnone { -; CHECK: pswapd +; CHECK: pswapd {{.*#+}} mm0 = mem[1,0] entry: %0 = bitcast <2 x i32> %a to x86_mmx %1 = tail call x86_mmx @llvm.x86.3dnowa.pswapd(x86_mmx %0) diff --git a/test/CodeGen/X86/GC/alloc_loop.ll b/test/CodeGen/X86/GC/alloc_loop.ll index 2a505e80aac8..b924e1cee069 100644 --- a/test/CodeGen/X86/GC/alloc_loop.ll +++ b/test/CodeGen/X86/GC/alloc_loop.ll @@ -1,4 +1,5 @@ ; RUN: llc < %s +; REQUIRES: default_triple declare i8* @llvm_gc_allocate(i32) diff --git a/test/CodeGen/X86/GC/cg-O0.ll b/test/CodeGen/X86/GC/cg-O0.ll index b4929425e94a..1a390c9eb1c1 100644 --- a/test/CodeGen/X86/GC/cg-O0.ll +++ b/test/CodeGen/X86/GC/cg-O0.ll @@ -1,4 +1,5 @@ ; RUN: llc < %s -O0 +; REQUIRES: default_triple define i32 @main() { entry: diff --git a/test/CodeGen/X86/GC/dynamic-frame-size.ll b/test/CodeGen/X86/GC/dynamic-frame-size.ll index a3583d46a29a..9ec9b8b08507 100644 --- a/test/CodeGen/X86/GC/dynamic-frame-size.ll +++ b/test/CodeGen/X86/GC/dynamic-frame-size.ll @@ -17,12 +17,12 @@ define void @test(i8* %ptr) gc "erlang" { ; CHECK: .note.gc ; CHECK-NEXT: .align 8 ; safe point count -; CHECK .short 1 -; CHECK .long .Ltmp0 +; CHECK: .short 1 +; CHECK: .long .Ltmp0 ; stack frame size (in words) -; CHECK .short -1 +; CHECK: .short -1 ; stack arity (arguments on the stack) -; CHECK .short 0 +; CHECK: .short 0 ; live root count -; CHECK .short 0 +; CHECK: .short 0 diff --git a/test/CodeGen/X86/GC/lower_gcroot.ll b/test/CodeGen/X86/GC/lower_gcroot.ll index c2d418ac50ef..8cccd78100f5 100644 --- a/test/CodeGen/X86/GC/lower_gcroot.ll +++ b/test/CodeGen/X86/GC/lower_gcroot.ll @@ -1,4 +1,5 @@ ; RUN: llc < %s +; REQUIRES: default_triple %Env = type i8* diff --git a/test/CodeGen/X86/MachineBranchProb.ll b/test/CodeGen/X86/MachineBranchProb.ll index 408c6b9151c3..ee1c658d4c55 100644 --- a/test/CodeGen/X86/MachineBranchProb.ll +++ b/test/CodeGen/X86/MachineBranchProb.ll @@ -18,9 +18,9 @@ for.cond2: ; preds = %for.inc, %for.cond %or.cond = or i1 %tobool, %cmp4 br i1 %or.cond, label %for.inc20, label %for.inc, !prof !0 ; CHECK: BB#1: derived from LLVM BB %for.cond2 -; CHECK: Successors according to CFG: BB#3(56008718) BB#4(3615818718) +; CHECK: Successors according to CFG: BB#3({{[0-9a-fx/= ]+}}1.53%) BB#4({{[0-9a-fx/= ]+}}98.47%) ; CHECK: BB#4: derived from LLVM BB %for.cond2 -; CHECK: Successors according to CFG: BB#3(56008718) BB#2(3559810000) +; CHECK: Successors according to CFG: BB#3({{[0-9a-fx/= ]+}}1.55%) BB#2({{[0-9a-fx/= ]+}}98.45%) for.inc: ; preds = %for.cond2 %shl = shl i32 %bit.0, 1 diff --git a/test/CodeGen/X86/MachineSink-DbgValue.ll b/test/CodeGen/X86/MachineSink-DbgValue.ll index 6f057c5f18e6..457d9beb37d5 100644 --- a/test/CodeGen/X86/MachineSink-DbgValue.ll +++ b/test/CodeGen/X86/MachineSink-DbgValue.ll @@ -3,7 +3,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64" target triple = "x86_64-apple-macosx10.7.0" -define i32 @foo(i32 %i, i32* nocapture %c) nounwind uwtable readonly ssp { +define i32 @foo(i32 %i, i32* nocapture %c) nounwind uwtable readonly ssp !dbg !1 { tail call void @llvm.dbg.value(metadata i32 %i, i64 0, metadata !6, metadata !DIExpression()), !dbg !12 %ab = load i32, i32* %c, align 1, !dbg !14 tail call void @llvm.dbg.value(metadata i32* %c, i64 0, metadata !7, metadata !DIExpression()), !dbg !13 @@ -28,17 +28,17 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnon !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!22} -!0 = !DICompileUnit(language: DW_LANG_C99, producer: "Apple clang version 3.0 (tags/Apple/clang-211.10.1) (based on LLVM 3.0svn)", isOptimized: true, emissionKind: 1, file: !20, enums: !21, retainedTypes: !21, subprograms: !18, imports: null) -!1 = !DISubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !20, scope: !2, type: !3, function: i32 (i32, i32*)* @foo, variables: !19) +!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "Apple clang version 3.0 (tags/Apple/clang-211.10.1) (based on LLVM 3.0svn)", isOptimized: true, emissionKind: 1, file: !20, enums: !21, retainedTypes: !21, subprograms: !18, imports: null) +!1 = distinct !DISubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, file: !20, scope: !2, type: !3, variables: !19) !2 = !DIFile(filename: "a.c", directory: "/private/tmp") !3 = !DISubroutineType(types: !4) !4 = !{!5} !5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) -!6 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "i", line: 2, arg: 1, scope: !1, file: !2, type: !5) -!7 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "c", line: 2, arg: 2, scope: !1, file: !2, type: !8) +!6 = !DILocalVariable(name: "i", line: 2, arg: 1, scope: !1, file: !2, type: !5) +!7 = !DILocalVariable(name: "c", line: 2, arg: 2, scope: !1, file: !2, type: !8) !8 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !0, baseType: !9) !9 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char) -!10 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 3, scope: !11, file: !2, type: !9) +!10 = !DILocalVariable(name: "a", line: 3, scope: !11, file: !2, type: !9) !11 = distinct !DILexicalBlock(line: 2, column: 25, file: !20, scope: !1) !12 = !DILocation(line: 2, column: 13, scope: !1) !13 = !DILocation(line: 2, column: 22, scope: !1) diff --git a/test/CodeGen/X86/MergeConsecutiveStores.ll b/test/CodeGen/X86/MergeConsecutiveStores.ll index c8f249b7529d..70af4184e8a2 100644 --- a/test/CodeGen/X86/MergeConsecutiveStores.ll +++ b/test/CodeGen/X86/MergeConsecutiveStores.ll @@ -1,13 +1,10 @@ -; RUN: llc -march=x86-64 -mcpu=corei7 -mattr=+avx < %s | FileCheck %s -; RUN: llc -march=x86-64 -mcpu=corei7 -mattr=+avx -addr-sink-using-gep=1 < %s | FileCheck %s - -target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" -target triple = "x86_64-apple-macosx10.8.0" +; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+avx < %s | FileCheck %s +; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+avx -addr-sink-using-gep=1 < %s | FileCheck %s %struct.A = type { i8, i8, i8, i8, i8, i8, i8, i8 } %struct.B = type { i32, i32, i32, i32, i32, i32, i32, i32 } -; CHECK: merge_const_store +; CHECK-LABEL: merge_const_store: ; save 1,2,3 ... as one big integer. ; CHECK: movabsq $578437695752307201 ; CHECK: ret @@ -42,7 +39,7 @@ define void @merge_const_store(i32 %count, %struct.A* nocapture %p) nounwind uwt } ; No vectors because we use noimplicitfloat -; CHECK: merge_const_store_no_vec +; CHECK-LABEL: merge_const_store_no_vec: ; CHECK-NOT: vmovups ; CHECK: ret define void @merge_const_store_no_vec(i32 %count, %struct.B* nocapture %p) noimplicitfloat{ @@ -76,7 +73,7 @@ define void @merge_const_store_no_vec(i32 %count, %struct.B* nocapture %p) noimp } ; Move the constants using a single vector store. -; CHECK: merge_const_store_vec +; CHECK-LABEL: merge_const_store_vec: ; CHECK: vmovups ; CHECK: ret define void @merge_const_store_vec(i32 %count, %struct.B* nocapture %p) nounwind uwtable noinline ssp { @@ -110,7 +107,7 @@ define void @merge_const_store_vec(i32 %count, %struct.B* nocapture %p) nounwind } ; Move the first 4 constants as a single vector. Move the rest as scalars. -; CHECK: merge_nonconst_store +; CHECK-LABEL: merge_nonconst_store: ; CHECK: movl $67305985 ; CHECK: movb ; CHECK: movb @@ -291,12 +288,16 @@ block4: ; preds = %4, %.lr.ph ret void } -;; On x86, even unaligned copies can be merged to vector ops. +;; On x86, even unaligned copies should be merged to vector ops. +;; TODO: however, this cannot happen at the moment, due to brokenness +;; in MergeConsecutiveStores. See UseAA FIXME in DAGCombiner.cpp +;; visitSTORE. + ; CHECK-LABEL: merge_loads_no_align: ; load: -; CHECK: vmovups +; CHECK-NOT: vmovups ;; TODO ; store: -; CHECK: vmovups +; CHECK-NOT: vmovups ;; TODO ; CHECK: ret define void @merge_loads_no_align(i32 %count, %struct.B* noalias nocapture %q, %struct.B* noalias nocapture %p) nounwind uwtable noinline ssp { %a1 = icmp sgt i32 %count, 0 @@ -335,7 +336,7 @@ block4: ; preds = %4, %.lr.ph ; Make sure that we merge the consecutive load/store sequence below and use a ; word (16 bit) instead of a byte copy. -; CHECK: MergeLoadStoreBaseIndexOffset +; CHECK-LABEL: MergeLoadStoreBaseIndexOffset: ; CHECK: movw (%{{.*}},%{{.*}}), [[REG:%[a-z]+]] ; CHECK: movw [[REG]], (%{{.*}}) define void @MergeLoadStoreBaseIndexOffset(i64* %a, i8* %b, i8* %c, i32 %n) { @@ -367,7 +368,7 @@ define void @MergeLoadStoreBaseIndexOffset(i64* %a, i8* %b, i8* %c, i32 %n) { ; Make sure that we merge the consecutive load/store sequence below and use a ; word (16 bit) instead of a byte copy even if there are intermediate sign ; extensions. -; CHECK: MergeLoadStoreBaseIndexOffsetSext +; CHECK-LABEL: MergeLoadStoreBaseIndexOffsetSext: ; CHECK: movw (%{{.*}},%{{.*}}), [[REG:%[a-z]+]] ; CHECK: movw [[REG]], (%{{.*}}) define void @MergeLoadStoreBaseIndexOffsetSext(i8* %a, i8* %b, i8* %c, i32 %n) { @@ -399,7 +400,7 @@ define void @MergeLoadStoreBaseIndexOffsetSext(i8* %a, i8* %b, i8* %c, i32 %n) { ; However, we can only merge ignore sign extensions when they are on all memory ; computations; -; CHECK: loadStoreBaseIndexOffsetSextNoSex +; CHECK-LABEL: loadStoreBaseIndexOffsetSextNoSex: ; CHECK-NOT: movw (%{{.*}},%{{.*}}), [[REG:%[a-z]+]] ; CHECK-NOT: movw [[REG]], (%{{.*}}) define void @loadStoreBaseIndexOffsetSextNoSex(i8* %a, i8* %b, i8* %c, i32 %n) { @@ -481,10 +482,8 @@ define void @merge_vec_extract_stores(<8 x float> %v1, <8 x float> %v2, <4 x flo ret void ; CHECK-LABEL: merge_vec_extract_stores -; CHECK: vmovaps %xmm0, 48(%rdi) -; CHECK-NEXT: vextractf128 $1, %ymm0, 64(%rdi) -; CHECK-NEXT: vmovaps %xmm1, 80(%rdi) -; CHECK-NEXT: vextractf128 $1, %ymm1, 96(%rdi) +; CHECK: vmovups %ymm0, 48(%rdi) +; CHECK-NEXT: vmovups %ymm1, 80(%rdi) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq } diff --git a/test/CodeGen/X86/StackColoring-dbg.ll b/test/CodeGen/X86/StackColoring-dbg.ll index 98c27f44fabc..91fe7f819383 100644 --- a/test/CodeGen/X86/StackColoring-dbg.ll +++ b/test/CodeGen/X86/StackColoring-dbg.ll @@ -27,9 +27,9 @@ declare void @llvm.lifetime.end(i64, i8* nocapture) nounwind !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!23} -!0 = !DICompileUnit(language: DW_LANG_C89, producer: "clang", isOptimized: true, emissionKind: 0, file: !1, enums: !{}, retainedTypes: !{}) +!0 = distinct !DICompileUnit(language: DW_LANG_C89, producer: "clang", isOptimized: true, emissionKind: 0, file: !1, enums: !{}, retainedTypes: !{}) !1 = !DIFile(filename: "t.c", directory: "") !16 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char) -!2 = !DISubprogram() -!22 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 16, scope: !2, file: !1, type: !16) +!2 = distinct !DISubprogram() +!22 = !DILocalVariable(name: "x", line: 16, scope: !2, file: !1, type: !16) !23 = !{i32 1, !"Debug Info Version", i32 3} diff --git a/test/CodeGen/X86/add-nsw-sext.ll b/test/CodeGen/X86/add-nsw-sext.ll new file mode 100644 index 000000000000..0a6f6c315c13 --- /dev/null +++ b/test/CodeGen/X86/add-nsw-sext.ll @@ -0,0 +1,168 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s + +; The fundamental problem: an add separated from other arithmetic by a sext can't +; be combined with the later instructions. However, if the first add is 'nsw', +; then we can promote the sext ahead of that add to allow optimizations. + +define i64 @add_nsw_consts(i32 %i) { +; CHECK-LABEL: add_nsw_consts: +; CHECK: # BB#0: +; CHECK-NEXT: movslq %edi, %rax +; CHECK-NEXT: addq $12, %rax +; CHECK-NEXT: retq + + %add = add nsw i32 %i, 5 + %ext = sext i32 %add to i64 + %idx = add i64 %ext, 7 + ret i64 %idx +} + +; An x86 bonus: If we promote the sext ahead of the 'add nsw', +; we allow LEA formation and eliminate an add instruction. + +define i64 @add_nsw_sext_add(i32 %i, i64 %x) { +; CHECK-LABEL: add_nsw_sext_add: +; CHECK: # BB#0: +; CHECK-NEXT: movslq %edi, %rax +; CHECK-NEXT: leaq 5(%rax,%rsi), %rax +; CHECK-NEXT: retq + + %add = add nsw i32 %i, 5 + %ext = sext i32 %add to i64 + %idx = add i64 %x, %ext + ret i64 %idx +} + +; Throw in a scale (left shift) because an LEA can do that too. +; Use a negative constant (LEA displacement) to verify that's handled correctly. + +define i64 @add_nsw_sext_lsh_add(i32 %i, i64 %x) { +; CHECK-LABEL: add_nsw_sext_lsh_add: +; CHECK: # BB#0: +; CHECK-NEXT: movslq %edi, %rax +; CHECK-NEXT: leaq -40(%rsi,%rax,8), %rax +; CHECK-NEXT: retq + + %add = add nsw i32 %i, -5 + %ext = sext i32 %add to i64 + %shl = shl i64 %ext, 3 + %idx = add i64 %x, %shl + ret i64 %idx +} + +; Don't promote the sext if it has no users. The wider add instruction needs an +; extra byte to encode. + +define i64 @add_nsw_sext(i32 %i, i64 %x) { +; CHECK-LABEL: add_nsw_sext: +; CHECK: # BB#0: +; CHECK-NEXT: addl $5, %edi +; CHECK-NEXT: movslq %edi, %rax +; CHECK-NEXT: retq + + %add = add nsw i32 %i, 5 + %ext = sext i32 %add to i64 + ret i64 %ext +} + +; The typical use case: a 64-bit system where an 'int' is used as an index into an array. + +define i8* @gep8(i32 %i, i8* %x) { +; CHECK-LABEL: gep8: +; CHECK: # BB#0: +; CHECK-NEXT: movslq %edi, %rax +; CHECK-NEXT: leaq 5(%rax,%rsi), %rax +; CHECK-NEXT: retq + + %add = add nsw i32 %i, 5 + %ext = sext i32 %add to i64 + %idx = getelementptr i8, i8* %x, i64 %ext + ret i8* %idx +} + +define i16* @gep16(i32 %i, i16* %x) { +; CHECK-LABEL: gep16: +; CHECK: # BB#0: +; CHECK-NEXT: movslq %edi, %rax +; CHECK-NEXT: leaq -10(%rsi,%rax,2), %rax +; CHECK-NEXT: retq + + %add = add nsw i32 %i, -5 + %ext = sext i32 %add to i64 + %idx = getelementptr i16, i16* %x, i64 %ext + ret i16* %idx +} + +define i32* @gep32(i32 %i, i32* %x) { +; CHECK-LABEL: gep32: +; CHECK: # BB#0: +; CHECK-NEXT: movslq %edi, %rax +; CHECK-NEXT: leaq 20(%rsi,%rax,4), %rax +; CHECK-NEXT: retq + + %add = add nsw i32 %i, 5 + %ext = sext i32 %add to i64 + %idx = getelementptr i32, i32* %x, i64 %ext + ret i32* %idx +} + +define i64* @gep64(i32 %i, i64* %x) { +; CHECK-LABEL: gep64: +; CHECK: # BB#0: +; CHECK-NEXT: movslq %edi, %rax +; CHECK-NEXT: leaq -40(%rsi,%rax,8), %rax +; CHECK-NEXT: retq + + %add = add nsw i32 %i, -5 + %ext = sext i32 %add to i64 + %idx = getelementptr i64, i64* %x, i64 %ext + ret i64* %idx +} + +; LEA can't scale by 16, but the adds can still be combined into an LEA. + +define i128* @gep128(i32 %i, i128* %x) { +; CHECK-LABEL: gep128: +; CHECK: # BB#0: +; CHECK-NEXT: movslq %edi, %rax +; CHECK-NEXT: shlq $4, %rax +; CHECK-NEXT: leaq 80(%rax,%rsi), %rax +; CHECK-NEXT: retq + + %add = add nsw i32 %i, 5 + %ext = sext i32 %add to i64 + %idx = getelementptr i128, i128* %x, i64 %ext + ret i128* %idx +} + +; A bigger win can be achieved when there is more than one use of the +; sign extended value. In this case, we can eliminate sign extension +; instructions plus use more efficient addressing modes for memory ops. + +define void @PR20134(i32* %a, i32 %i) { +; CHECK-LABEL: PR20134: +; CHECK: # BB#0: +; CHECK-NEXT: movslq %esi, %rax +; CHECK-NEXT: movl 4(%rdi,%rax,4), %ecx +; CHECK-NEXT: addl 8(%rdi,%rax,4), %ecx +; CHECK-NEXT: movl %ecx, (%rdi,%rax,4) +; CHECK-NEXT: retq + + %add1 = add nsw i32 %i, 1 + %idx1 = sext i32 %add1 to i64 + %gep1 = getelementptr i32, i32* %a, i64 %idx1 + %load1 = load i32, i32* %gep1, align 4 + + %add2 = add nsw i32 %i, 2 + %idx2 = sext i32 %add2 to i64 + %gep2 = getelementptr i32, i32* %a, i64 %idx2 + %load2 = load i32, i32* %gep2, align 4 + + %add3 = add i32 %load1, %load2 + %idx3 = sext i32 %i to i64 + %gep3 = getelementptr i32, i32* %a, i64 %idx3 + store i32 %add3, i32* %gep3, align 4 + ret void +} + diff --git a/test/CodeGen/X86/aliases.ll b/test/CodeGen/X86/aliases.ll index 3f19a064323c..50c7b929c827 100644 --- a/test/CodeGen/X86/aliases.ll +++ b/test/CodeGen/X86/aliases.ll @@ -2,7 +2,7 @@ ; RUN: -relocation-model=pic | FileCheck %s @thread_var = thread_local global i32 42, align 4 -@thread_alias = thread_local(localdynamic) alias i32* @thread_var +@thread_alias = thread_local(localdynamic) alias i32, i32* @thread_var ; CHECK-LABEL: get_thread_var define i32* @get_thread_var() { @@ -19,10 +19,10 @@ define i32* @get_thread_alias() { @bar = global i32 42 ; CHECK-DAG: .globl foo1 -@foo1 = alias i32* @bar +@foo1 = alias i32, i32* @bar ; CHECK-DAG: .globl foo2 -@foo2 = alias i32* @bar +@foo2 = alias i32, i32* @bar %FunTy = type i32() @@ -30,35 +30,35 @@ define i32 @foo_f() { ret i32 0 } ; CHECK-DAG: .weak bar_f -@bar_f = weak alias %FunTy* @foo_f +@bar_f = weak alias %FunTy, %FunTy* @foo_f -@bar_l = linkonce_odr alias i32* @bar +@bar_l = linkonce_odr alias i32, i32* @bar ; CHECK-DAG: .weak bar_l -@bar_i = internal alias i32* @bar +@bar_i = internal alias i32, i32* @bar ; CHECK-DAG: .globl A -@A = alias bitcast (i32* @bar to i64*) +@A = alias i64, bitcast (i32* @bar to i64*) ; CHECK-DAG: .globl bar_h ; CHECK-DAG: .hidden bar_h -@bar_h = hidden alias i32* @bar +@bar_h = hidden alias i32, i32* @bar ; CHECK-DAG: .globl bar_p ; CHECK-DAG: .protected bar_p -@bar_p = protected alias i32* @bar +@bar_p = protected alias i32, i32* @bar ; CHECK-DAG: test2 = bar+4 -@test2 = alias getelementptr(i32, i32 *@bar, i32 1) +@test2 = alias i32, getelementptr(i32, i32* @bar, i32 1) ; CHECK-DAG: test3 = 42 -@test3 = alias inttoptr(i32 42 to i32*) +@test3 = alias i32, inttoptr(i32 42 to i32*) ; CHECK-DAG: test4 = bar -@test4 = alias inttoptr(i64 ptrtoint (i32* @bar to i64) to i32*) +@test4 = alias i32, inttoptr(i64 ptrtoint (i32* @bar to i64) to i32*) ; CHECK-DAG: test5 = test2-bar -@test5 = alias inttoptr(i32 sub (i32 ptrtoint (i32* @test2 to i32), +@test5 = alias i32, inttoptr(i32 sub (i32 ptrtoint (i32* @test2 to i32), i32 ptrtoint (i32* @bar to i32)) to i32*) ; CHECK-DAG: .globl test diff --git a/test/CodeGen/X86/and-encoding.ll b/test/CodeGen/X86/and-encoding.ll new file mode 100644 index 000000000000..f7bbac2a4bd9 --- /dev/null +++ b/test/CodeGen/X86/and-encoding.ll @@ -0,0 +1,41 @@ +; RUN: llc -show-mc-encoding < %s | FileCheck %s + +; Test that the direct object emission selects the and variant with 8 bit +; immediate. +; We used to get this wrong when using direct object emission, but not when +; reading assembly. + + +target triple = "x86_64-pc-linux" + +define void @f1() { +; CHECK-LABEL: f1: +; CHECK: andq $-32, %rsp # encoding: [0x48,0x83,0xe4,0xe0] + %foo = alloca i8, align 32 + ret void +} + +define void @f2(i1 *%x, i16 *%y) { +; CHECK-LABEL: f2: +; CHECK: andl $1, %eax # encoding: [0x83,0xe0,0x01] + %a = load i1, i1* %x + %b = zext i1 %a to i16 + store i16 %b, i16* %y + ret void +} + +define i32 @f3(i1 *%x) { +; CHECK-LABEL: f3: +; CHECK: andl $1, %eax # encoding: [0x83,0xe0,0x01] + %a = load i1, i1* %x + %b = zext i1 %a to i32 + ret i32 %b +} + +define i64 @f4(i1 *%x) { +; CHECK-LABEL: f4: +; CHECK: andl $1, %eax # encoding: [0x83,0xe0,0x01] + %a = load i1, i1* %x + %b = zext i1 %a to i64 + ret i64 %b +} diff --git a/test/CodeGen/X86/atomic-flags.ll b/test/CodeGen/X86/atomic-flags.ll new file mode 100644 index 000000000000..e0c4a915965c --- /dev/null +++ b/test/CodeGen/X86/atomic-flags.ll @@ -0,0 +1,61 @@ +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -verify-machineinstrs | FileCheck %s +; RUN: llc < %s -mtriple=i686-unknown-unknown -verify-machineinstrs | FileCheck %s + +; Make sure that flags are properly preserved despite atomic optimizations. + +define i32 @atomic_and_flags_1(i8* %p, i32 %a, i32 %b) { +; CHECK-LABEL: atomic_and_flags_1: + + ; Generate flags value, and use it. + ; CHECK: cmpl + ; CHECK-NEXT: jne + %cmp = icmp eq i32 %a, %b + br i1 %cmp, label %L1, label %L2 + +L1: + ; The following pattern will get folded. + ; CHECK: incb + %1 = load atomic i8, i8* %p seq_cst, align 1 + %2 = add i8 %1, 1 ; This forces the INC instruction to be generated. + store atomic i8 %2, i8* %p release, align 1 + + ; Use the comparison result again. We need to rematerialize the comparison + ; somehow. This test checks that cmpl gets emitted again, but any + ; rematerialization would work (the optimizer used to clobber the flags with + ; the add). + ; CHECK-NEXT: cmpl + ; CHECK-NEXT: jne + br i1 %cmp, label %L3, label %L4 + +L2: + ret i32 2 + +L3: + ret i32 3 + +L4: + ret i32 4 +} + +; Same as above, but using 2 as immediate to avoid the INC instruction. +define i32 @atomic_and_flags_2(i8* %p, i32 %a, i32 %b) { +; CHECK-LABEL: atomic_and_flags_2: + ; CHECK: cmpl + ; CHECK-NEXT: jne + %cmp = icmp eq i32 %a, %b + br i1 %cmp, label %L1, label %L2 +L1: + ; CHECK: addb + %1 = load atomic i8, i8* %p seq_cst, align 1 + %2 = add i8 %1, 2 + store atomic i8 %2, i8* %p release, align 1 + ; CHECK-NEXT: cmpl + ; CHECK-NEXT: jne + br i1 %cmp, label %L3, label %L4 +L2: + ret i32 2 +L3: + ret i32 3 +L4: + ret i32 4 +} diff --git a/test/CodeGen/X86/atomic-minmax-i6432.ll b/test/CodeGen/X86/atomic-minmax-i6432.ll index 4989bc14ef86..d5d3fa6db5e8 100644 --- a/test/CodeGen/X86/atomic-minmax-i6432.ll +++ b/test/CodeGen/X86/atomic-minmax-i6432.ll @@ -8,7 +8,7 @@ define void @atomic_maxmin_i6432() { %1 = atomicrmw max i64* @sc64, i64 5 acquire ; LINUX: [[LABEL:.LBB[0-9]+_[0-9]+]] ; LINUX: cmpl -; LINUX: seta +; LINUX: sbbl ; LINUX: cmovne ; LINUX: cmovne ; LINUX: lock cmpxchg8b @@ -16,7 +16,7 @@ define void @atomic_maxmin_i6432() { %2 = atomicrmw min i64* @sc64, i64 6 acquire ; LINUX: [[LABEL:.LBB[0-9]+_[0-9]+]] ; LINUX: cmpl -; LINUX: setb +; LINUX: sbbl ; LINUX: cmovne ; LINUX: cmovne ; LINUX: lock cmpxchg8b @@ -24,7 +24,7 @@ define void @atomic_maxmin_i6432() { %3 = atomicrmw umax i64* @sc64, i64 7 acquire ; LINUX: [[LABEL:.LBB[0-9]+_[0-9]+]] ; LINUX: cmpl -; LINUX: seta +; LINUX: sbbl ; LINUX: cmovne ; LINUX: cmovne ; LINUX: lock cmpxchg8b @@ -32,7 +32,7 @@ define void @atomic_maxmin_i6432() { %4 = atomicrmw umin i64* @sc64, i64 8 acquire ; LINUX: [[LABEL:.LBB[0-9]+_[0-9]+]] ; LINUX: cmpl -; LINUX: setb +; LINUX: sbbl ; LINUX: cmovne ; LINUX: cmovne ; LINUX: lock cmpxchg8b diff --git a/test/CodeGen/X86/atomic-non-integer.ll b/test/CodeGen/X86/atomic-non-integer.ll new file mode 100644 index 000000000000..98fcd96d3e4c --- /dev/null +++ b/test/CodeGen/X86/atomic-non-integer.ll @@ -0,0 +1,108 @@ +; RUN: llc < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=sse2 | FileCheck %s + +; Note: This test is testing that the lowering for atomics matches what we +; currently emit for non-atomics + the atomic restriction. The presence of +; particular lowering detail in these tests should not be read as requiring +; that detail for correctness unless it's related to the atomicity itself. +; (Specifically, there were reviewer questions about the lowering for halfs +; and their calling convention which remain unresolved.) + +define void @store_half(half* %fptr, half %v) { +; CHECK-LABEL: @store_half +; CHECK: movq %rdi, %rbx +; CHECK: callq __gnu_f2h_ieee +; CHECK: movw %ax, (%rbx) + store atomic half %v, half* %fptr unordered, align 2 + ret void +} + +define void @store_float(float* %fptr, float %v) { +; CHECK-LABEL: @store_float +; CHECK: movd %xmm0, %eax +; CHECK: movl %eax, (%rdi) + store atomic float %v, float* %fptr unordered, align 4 + ret void +} + +define void @store_double(double* %fptr, double %v) { +; CHECK-LABEL: @store_double +; CHECK: movd %xmm0, %rax +; CHECK: movq %rax, (%rdi) + store atomic double %v, double* %fptr unordered, align 8 + ret void +} + +define void @store_fp128(fp128* %fptr, fp128 %v) { +; CHECK-LABEL: @store_fp128 +; CHECK: callq __sync_lock_test_and_set_16 + store atomic fp128 %v, fp128* %fptr unordered, align 16 + ret void +} + +define half @load_half(half* %fptr) { +; CHECK-LABEL: @load_half +; CHECK: movw (%rdi), %ax +; CHECK: movzwl %ax, %edi +; CHECK: jmp __gnu_h2f_ieee + %v = load atomic half, half* %fptr unordered, align 2 + ret half %v +} + +define float @load_float(float* %fptr) { +; CHECK-LABEL: @load_float +; CHECK: movl (%rdi), %eax +; CHECK: movd %eax, %xmm0 + %v = load atomic float, float* %fptr unordered, align 4 + ret float %v +} + +define double @load_double(double* %fptr) { +; CHECK-LABEL: @load_double +; CHECK: movq (%rdi), %rax +; CHECK: movd %rax, %xmm0 + %v = load atomic double, double* %fptr unordered, align 8 + ret double %v +} + +define fp128 @load_fp128(fp128* %fptr) { +; CHECK-LABEL: @load_fp128 +; CHECK: callq __sync_val_compare_and_swap_16 + %v = load atomic fp128, fp128* %fptr unordered, align 16 + ret fp128 %v +} + + +; sanity check the seq_cst lowering since that's the +; interesting one from an ordering perspective on x86. + +define void @store_float_seq_cst(float* %fptr, float %v) { +; CHECK-LABEL: @store_float_seq_cst +; CHECK: movd %xmm0, %eax +; CHECK: xchgl %eax, (%rdi) + store atomic float %v, float* %fptr seq_cst, align 4 + ret void +} + +define void @store_double_seq_cst(double* %fptr, double %v) { +; CHECK-LABEL: @store_double_seq_cst +; CHECK: movd %xmm0, %rax +; CHECK: xchgq %rax, (%rdi) + store atomic double %v, double* %fptr seq_cst, align 8 + ret void +} + +define float @load_float_seq_cst(float* %fptr) { +; CHECK-LABEL: @load_float_seq_cst +; CHECK: movl (%rdi), %eax +; CHECK: movd %eax, %xmm0 + %v = load atomic float, float* %fptr seq_cst, align 4 + ret float %v +} + +define double @load_double_seq_cst(double* %fptr) { +; CHECK-LABEL: @load_double_seq_cst +; CHECK: movq (%rdi), %rax +; CHECK: movd %rax, %xmm0 + %v = load atomic double, double* %fptr seq_cst, align 8 + ret double %v +} diff --git a/test/CodeGen/X86/atomic128.ll b/test/CodeGen/X86/atomic128.ll index dea7d482f989..c41269b0b606 100644 --- a/test/CodeGen/X86/atomic128.ll +++ b/test/CodeGen/X86/atomic128.ll @@ -119,16 +119,9 @@ define void @fetch_and_min(i128* %p, i128 %bits) { ; CHECK-DAG: movq 8(%rdi), %rdx ; CHECK: [[LOOP:.?LBB[0-9]+_[0-9]+]]: -; CHECK: cmpq %rsi, %rax -; CHECK: setbe [[CMP:%[a-z0-9]+]] -; CHECK: cmpq [[INCHI]], %rdx -; CHECK: setle [[HICMP:%[a-z0-9]+]] -; CHECK: je [[USE_LO:.?LBB[0-9]+_[0-9]+]] - -; CHECK: movb [[HICMP]], [[CMP]] -; CHECK: [[USE_LO]]: -; CHECK: testb [[CMP]], [[CMP]] -; CHECK: movq %rsi, %rbx +; CHECK: cmpq +; CHECK: sbbq +; CHECK: setg ; CHECK: cmovneq %rax, %rbx ; CHECK: movq [[INCHI]], %rcx ; CHECK: cmovneq %rdx, %rcx @@ -151,16 +144,9 @@ define void @fetch_and_max(i128* %p, i128 %bits) { ; CHECK-DAG: movq 8(%rdi), %rdx ; CHECK: [[LOOP:.?LBB[0-9]+_[0-9]+]]: -; CHECK: cmpq %rsi, %rax -; CHECK: setae [[CMP:%[a-z0-9]+]] -; CHECK: cmpq [[INCHI]], %rdx -; CHECK: setge [[HICMP:%[a-z0-9]+]] -; CHECK: je [[USE_LO:.?LBB[0-9]+_[0-9]+]] - -; CHECK: movb [[HICMP]], [[CMP]] -; CHECK: [[USE_LO]]: -; CHECK: testb [[CMP]], [[CMP]] -; CHECK: movq %rsi, %rbx +; CHECK: cmpq +; CHECK: sbbq +; CHECK: setge ; CHECK: cmovneq %rax, %rbx ; CHECK: movq [[INCHI]], %rcx ; CHECK: cmovneq %rdx, %rcx @@ -183,16 +169,9 @@ define void @fetch_and_umin(i128* %p, i128 %bits) { ; CHECK-DAG: movq 8(%rdi), %rdx ; CHECK: [[LOOP:.?LBB[0-9]+_[0-9]+]]: -; CHECK: cmpq %rsi, %rax -; CHECK: setbe [[CMP:%[a-z0-9]+]] -; CHECK: cmpq [[INCHI]], %rdx -; CHECK: setbe [[HICMP:%[a-z0-9]+]] -; CHECK: je [[USE_LO:.?LBB[0-9]+_[0-9]+]] - -; CHECK: movb [[HICMP]], [[CMP]] -; CHECK: [[USE_LO]]: -; CHECK: testb [[CMP]], [[CMP]] -; CHECK: movq %rsi, %rbx +; CHECK: cmpq +; CHECK: sbbq +; CHECK: seta ; CHECK: cmovneq %rax, %rbx ; CHECK: movq [[INCHI]], %rcx ; CHECK: cmovneq %rdx, %rcx @@ -215,16 +194,9 @@ define void @fetch_and_umax(i128* %p, i128 %bits) { ; CHECK-DAG: movq 8(%rdi), %rdx ; CHECK: [[LOOP:.?LBB[0-9]+_[0-9]+]]: -; CHECK: cmpq %rax, %rsi -; CHECK: setb [[CMP:%[a-z0-9]+]] -; CHECK: cmpq [[INCHI]], %rdx -; CHECK: seta [[HICMP:%[a-z0-9]+]] -; CHECK: je [[USE_LO:.?LBB[0-9]+_[0-9]+]] - -; CHECK: movb [[HICMP]], [[CMP]] -; CHECK: [[USE_LO]]: -; CHECK: testb [[CMP]], [[CMP]] -; CHECK: movq %rsi, %rbx +; CHECK: cmpq +; CHECK: sbbq +; CHECK: setb ; CHECK: cmovneq %rax, %rbx ; CHECK: movq [[INCHI]], %rcx ; CHECK: cmovneq %rdx, %rcx diff --git a/test/CodeGen/X86/atomic_mi.ll b/test/CodeGen/X86/atomic_mi.ll index 7a6204fc8930..356d9dcff6fa 100644 --- a/test/CodeGen/X86/atomic_mi.ll +++ b/test/CodeGen/X86/atomic_mi.ll @@ -1,6 +1,6 @@ -; RUN: llc < %s -march=x86-64 -verify-machineinstrs | FileCheck %s --check-prefix X64 -; RUN: llc < %s -march=x86 -verify-machineinstrs | FileCheck %s --check-prefix X32 -; RUN: llc < %s -march=x86-64 -mattr=slow-incdec -verify-machineinstrs | FileCheck %s --check-prefix SLOW_INC +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -verify-machineinstrs | FileCheck %s --check-prefix X64 +; RUN: llc < %s -mtriple=i686-unknown-unknown -verify-machineinstrs | FileCheck %s --check-prefix X32 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=slow-incdec -verify-machineinstrs | FileCheck %s --check-prefix SLOW_INC ; This file checks that atomic (non-seq_cst) stores of immediate values are ; done in one mov instruction and not 2. More precisely, it makes sure that the @@ -14,7 +14,11 @@ ; The binary operations supported are currently add, and, or, xor. ; sub is not supported because they are translated by an addition of the ; negated immediate. -; Finally, we also check the same kind of pattern for inc/dec +; +; We also check the same patterns: +; - For inc/dec. +; - For register instead of immediate operands. +; - For floating point operations. ; seq_cst stores are left as (lock) xchgl, but we try to check every other ; attribute at least once. @@ -25,10 +29,10 @@ ; an implicit lock prefix, so making it explicit is not required. define void @store_atomic_imm_8(i8* %p) { -; X64-LABEL: store_atomic_imm_8 +; X64-LABEL: store_atomic_imm_8: ; X64: movb ; X64-NOT: movb -; X32-LABEL: store_atomic_imm_8 +; X32-LABEL: store_atomic_imm_8: ; X32: movb ; X32-NOT: movb store atomic i8 42, i8* %p release, align 1 @@ -36,10 +40,10 @@ define void @store_atomic_imm_8(i8* %p) { } define void @store_atomic_imm_16(i16* %p) { -; X64-LABEL: store_atomic_imm_16 +; X64-LABEL: store_atomic_imm_16: ; X64: movw ; X64-NOT: movw -; X32-LABEL: store_atomic_imm_16 +; X32-LABEL: store_atomic_imm_16: ; X32: movw ; X32-NOT: movw store atomic i16 42, i16* %p monotonic, align 2 @@ -47,12 +51,12 @@ define void @store_atomic_imm_16(i16* %p) { } define void @store_atomic_imm_32(i32* %p) { -; X64-LABEL: store_atomic_imm_32 +; X64-LABEL: store_atomic_imm_32: ; X64: movl ; X64-NOT: movl ; On 32 bits, there is an extra movl for each of those functions ; (probably for alignment reasons). -; X32-LABEL: store_atomic_imm_32 +; X32-LABEL: store_atomic_imm_32: ; X32: movl 4(%esp), %eax ; X32: movl ; X32-NOT: movl @@ -61,12 +65,12 @@ define void @store_atomic_imm_32(i32* %p) { } define void @store_atomic_imm_64(i64* %p) { -; X64-LABEL: store_atomic_imm_64 +; X64-LABEL: store_atomic_imm_64: ; X64: movq ; X64-NOT: movq ; These are implemented with a CAS loop on 32 bit architectures, and thus ; cannot be optimized in the same way as the others. -; X32-LABEL: store_atomic_imm_64 +; X32-LABEL: store_atomic_imm_64: ; X32: cmpxchg8b store atomic i64 42, i64* %p release, align 8 ret void @@ -75,7 +79,7 @@ define void @store_atomic_imm_64(i64* %p) { ; If an immediate is too big to fit in 32 bits, it cannot be store in one mov, ; even on X64, one must use movabsq that can only target a register. define void @store_atomic_imm_64_big(i64* %p) { -; X64-LABEL: store_atomic_imm_64_big +; X64-LABEL: store_atomic_imm_64_big: ; X64: movabsq ; X64: movq store atomic i64 100000000000, i64* %p monotonic, align 8 @@ -84,9 +88,9 @@ define void @store_atomic_imm_64_big(i64* %p) { ; It would be incorrect to replace a lock xchgl by a movl define void @store_atomic_imm_32_seq_cst(i32* %p) { -; X64-LABEL: store_atomic_imm_32_seq_cst +; X64-LABEL: store_atomic_imm_32_seq_cst: ; X64: xchgl -; X32-LABEL: store_atomic_imm_32_seq_cst +; X32-LABEL: store_atomic_imm_32_seq_cst: ; X32: xchgl store atomic i32 42, i32* %p seq_cst, align 4 ret void @@ -94,12 +98,12 @@ define void @store_atomic_imm_32_seq_cst(i32* %p) { ; ----- ADD ----- -define void @add_8(i8* %p) { -; X64-LABEL: add_8 +define void @add_8i(i8* %p) { +; X64-LABEL: add_8i: ; X64-NOT: lock ; X64: addb ; X64-NOT: movb -; X32-LABEL: add_8 +; X32-LABEL: add_8i: ; X32-NOT: lock ; X32: addb ; X32-NOT: movb @@ -109,12 +113,27 @@ define void @add_8(i8* %p) { ret void } -define void @add_16(i16* %p) { +define void @add_8r(i8* %p, i8 %v) { +; X64-LABEL: add_8r: +; X64-NOT: lock +; X64: addb +; X64-NOT: movb +; X32-LABEL: add_8r: +; X32-NOT: lock +; X32: addb +; X32-NOT: movb + %1 = load atomic i8, i8* %p seq_cst, align 1 + %2 = add i8 %1, %v + store atomic i8 %2, i8* %p release, align 1 + ret void +} + +define void @add_16i(i16* %p) { ; Currently the transformation is not done on 16 bit accesses, as the backend ; treat 16 bit arithmetic as expensive on X86/X86_64. -; X64-LABEL: add_16 +; X64-LABEL: add_16i: ; X64-NOT: addw -; X32-LABEL: add_16 +; X32-LABEL: add_16i: ; X32-NOT: addw %1 = load atomic i16, i16* %p acquire, align 2 %2 = add i16 %1, 2 @@ -122,12 +141,25 @@ define void @add_16(i16* %p) { ret void } -define void @add_32(i32* %p) { -; X64-LABEL: add_32 +define void @add_16r(i16* %p, i16 %v) { +; Currently the transformation is not done on 16 bit accesses, as the backend +; treat 16 bit arithmetic as expensive on X86/X86_64. +; X64-LABEL: add_16r: +; X64-NOT: addw +; X32-LABEL: add_16r: +; X32-NOT: addw [.*], ( + %1 = load atomic i16, i16* %p acquire, align 2 + %2 = add i16 %1, %v + store atomic i16 %2, i16* %p release, align 2 + ret void +} + +define void @add_32i(i32* %p) { +; X64-LABEL: add_32i: ; X64-NOT: lock ; X64: addl ; X64-NOT: movl -; X32-LABEL: add_32 +; X32-LABEL: add_32i: ; X32-NOT: lock ; X32: addl ; X32-NOT: movl @@ -137,23 +169,94 @@ define void @add_32(i32* %p) { ret void } -define void @add_64(i64* %p) { -; X64-LABEL: add_64 +define void @add_32r(i32* %p, i32 %v) { +; X64-LABEL: add_32r: +; X64-NOT: lock +; X64: addl +; X64-NOT: movl +; X32-LABEL: add_32r: +; X32-NOT: lock +; X32: addl +; X32-NOT: movl + %1 = load atomic i32, i32* %p acquire, align 4 + %2 = add i32 %1, %v + store atomic i32 %2, i32* %p monotonic, align 4 + ret void +} + +; The following is a corner case where the load is added to itself. The pattern +; matching should not fold this. We only test with 32-bit add, but the same +; applies to other sizes and operations. +define void @add_32r_self(i32* %p) { +; X64-LABEL: add_32r_self: +; X64-NOT: lock +; X64: movl (%[[M:[a-z]+]]), %[[R:[a-z]+]] +; X64: addl %[[R]], %[[R]] +; X64: movl %[[R]], (%[[M]]) +; X32-LABEL: add_32r_self: +; X32-NOT: lock +; X32: movl (%[[M:[a-z]+]]), %[[R:[a-z]+]] +; X32: addl %[[R]], %[[R]] +; X32: movl %[[R]], (%[[M]]) + %1 = load atomic i32, i32* %p acquire, align 4 + %2 = add i32 %1, %1 + store atomic i32 %2, i32* %p monotonic, align 4 + ret void +} + +; The following is a corner case where the load's result is returned. The +; optimizer isn't allowed to duplicate the load because it's atomic. +define i32 @add_32r_ret_load(i32* %p, i32 %v) { +; X64-LABEL: add_32r_ret_load: +; X64-NOT: lock +; X64: movl (%rdi), %eax +; X64-NEXT: addl %eax, %esi +; X64-NEXT: movl %esi, (%rdi) +; X64-NEXT: retq +; X32-LABEL: add_32r_ret_load: +; X32-NOT: lock +; X32: movl 4(%esp), %[[P:[a-z]+]] +; X32-NEXT: movl (%[[P]]), +; X32-NOT: %[[P]] +; More code here, we just don't want it to load from P. +; X32: movl %{{.*}}, (%[[P]]) +; X32-NEXT: retl + %1 = load atomic i32, i32* %p acquire, align 4 + %2 = add i32 %1, %v + store atomic i32 %2, i32* %p monotonic, align 4 + ret i32 %1 +} + +define void @add_64i(i64* %p) { +; X64-LABEL: add_64i: ; X64-NOT: lock ; X64: addq ; X64-NOT: movq ; We do not check X86-32 as it cannot do 'addq'. -; X32-LABEL: add_64 +; X32-LABEL: add_64i: %1 = load atomic i64, i64* %p acquire, align 8 %2 = add i64 %1, 2 store atomic i64 %2, i64* %p release, align 8 ret void } -define void @add_32_seq_cst(i32* %p) { -; X64-LABEL: add_32_seq_cst +define void @add_64r(i64* %p, i64 %v) { +; X64-LABEL: add_64r: +; X64-NOT: lock +; X64: addq +; X64-NOT: movq +; We do not check X86-32 as it cannot do 'addq'. +; X32-LABEL: add_64r: + %1 = load atomic i64, i64* %p acquire, align 8 + %2 = add i64 %1, %v + store atomic i64 %2, i64* %p release, align 8 + ret void +} + +define void @add_32i_seq_cst(i32* %p) { +; X64-LABEL: add_32i_seq_cst: ; X64: xchgl -; X32-LABEL: add_32_seq_cst +; X32-LABEL: add_32i_seq_cst: ; X32: xchgl %1 = load atomic i32, i32* %p monotonic, align 4 %2 = add i32 %1, 2 @@ -161,14 +264,25 @@ define void @add_32_seq_cst(i32* %p) { ret void } +define void @add_32r_seq_cst(i32* %p, i32 %v) { +; X64-LABEL: add_32r_seq_cst: +; X64: xchgl +; X32-LABEL: add_32r_seq_cst: +; X32: xchgl + %1 = load atomic i32, i32* %p monotonic, align 4 + %2 = add i32 %1, %v + store atomic i32 %2, i32* %p seq_cst, align 4 + ret void +} + ; ----- AND ----- -define void @and_8(i8* %p) { -; X64-LABEL: and_8 +define void @and_8i(i8* %p) { +; X64-LABEL: and_8i: ; X64-NOT: lock ; X64: andb ; X64-NOT: movb -; X32-LABEL: and_8 +; X32-LABEL: and_8i: ; X32-NOT: lock ; X32: andb ; X32-NOT: movb @@ -178,12 +292,27 @@ define void @and_8(i8* %p) { ret void } -define void @and_16(i16* %p) { +define void @and_8r(i8* %p, i8 %v) { +; X64-LABEL: and_8r: +; X64-NOT: lock +; X64: andb +; X64-NOT: movb +; X32-LABEL: and_8r: +; X32-NOT: lock +; X32: andb +; X32-NOT: movb + %1 = load atomic i8, i8* %p monotonic, align 1 + %2 = and i8 %1, %v + store atomic i8 %2, i8* %p release, align 1 + ret void +} + +define void @and_16i(i16* %p) { ; Currently the transformation is not done on 16 bit accesses, as the backend ; treat 16 bit arithmetic as expensive on X86/X86_64. -; X64-LABEL: and_16 +; X64-LABEL: and_16i: ; X64-NOT: andw -; X32-LABEL: and_16 +; X32-LABEL: and_16i: ; X32-NOT: andw %1 = load atomic i16, i16* %p acquire, align 2 %2 = and i16 %1, 2 @@ -191,12 +320,25 @@ define void @and_16(i16* %p) { ret void } -define void @and_32(i32* %p) { -; X64-LABEL: and_32 +define void @and_16r(i16* %p, i16 %v) { +; Currently the transformation is not done on 16 bit accesses, as the backend +; treat 16 bit arithmetic as expensive on X86/X86_64. +; X64-LABEL: and_16r: +; X64-NOT: andw +; X32-LABEL: and_16r: +; X32-NOT: andw [.*], ( + %1 = load atomic i16, i16* %p acquire, align 2 + %2 = and i16 %1, %v + store atomic i16 %2, i16* %p release, align 2 + ret void +} + +define void @and_32i(i32* %p) { +; X64-LABEL: and_32i: ; X64-NOT: lock ; X64: andl ; X64-NOT: movl -; X32-LABEL: and_32 +; X32-LABEL: and_32i: ; X32-NOT: lock ; X32: andl ; X32-NOT: movl @@ -206,23 +348,51 @@ define void @and_32(i32* %p) { ret void } -define void @and_64(i64* %p) { -; X64-LABEL: and_64 +define void @and_32r(i32* %p, i32 %v) { +; X64-LABEL: and_32r: +; X64-NOT: lock +; X64: andl +; X64-NOT: movl +; X32-LABEL: and_32r: +; X32-NOT: lock +; X32: andl +; X32-NOT: movl + %1 = load atomic i32, i32* %p acquire, align 4 + %2 = and i32 %1, %v + store atomic i32 %2, i32* %p release, align 4 + ret void +} + +define void @and_64i(i64* %p) { +; X64-LABEL: and_64i: ; X64-NOT: lock ; X64: andq ; X64-NOT: movq ; We do not check X86-32 as it cannot do 'andq'. -; X32-LABEL: and_64 +; X32-LABEL: and_64i: %1 = load atomic i64, i64* %p acquire, align 8 %2 = and i64 %1, 2 store atomic i64 %2, i64* %p release, align 8 ret void } -define void @and_32_seq_cst(i32* %p) { -; X64-LABEL: and_32_seq_cst +define void @and_64r(i64* %p, i64 %v) { +; X64-LABEL: and_64r: +; X64-NOT: lock +; X64: andq +; X64-NOT: movq +; We do not check X86-32 as it cannot do 'andq'. +; X32-LABEL: and_64r: + %1 = load atomic i64, i64* %p acquire, align 8 + %2 = and i64 %1, %v + store atomic i64 %2, i64* %p release, align 8 + ret void +} + +define void @and_32i_seq_cst(i32* %p) { +; X64-LABEL: and_32i_seq_cst: ; X64: xchgl -; X32-LABEL: and_32_seq_cst +; X32-LABEL: and_32i_seq_cst: ; X32: xchgl %1 = load atomic i32, i32* %p monotonic, align 4 %2 = and i32 %1, 2 @@ -230,14 +400,25 @@ define void @and_32_seq_cst(i32* %p) { ret void } +define void @and_32r_seq_cst(i32* %p, i32 %v) { +; X64-LABEL: and_32r_seq_cst: +; X64: xchgl +; X32-LABEL: and_32r_seq_cst: +; X32: xchgl + %1 = load atomic i32, i32* %p monotonic, align 4 + %2 = and i32 %1, %v + store atomic i32 %2, i32* %p seq_cst, align 4 + ret void +} + ; ----- OR ----- -define void @or_8(i8* %p) { -; X64-LABEL: or_8 +define void @or_8i(i8* %p) { +; X64-LABEL: or_8i: ; X64-NOT: lock ; X64: orb ; X64-NOT: movb -; X32-LABEL: or_8 +; X32-LABEL: or_8i: ; X32-NOT: lock ; X32: orb ; X32-NOT: movb @@ -247,10 +428,25 @@ define void @or_8(i8* %p) { ret void } -define void @or_16(i16* %p) { -; X64-LABEL: or_16 +define void @or_8r(i8* %p, i8 %v) { +; X64-LABEL: or_8r: +; X64-NOT: lock +; X64: orb +; X64-NOT: movb +; X32-LABEL: or_8r: +; X32-NOT: lock +; X32: orb +; X32-NOT: movb + %1 = load atomic i8, i8* %p acquire, align 1 + %2 = or i8 %1, %v + store atomic i8 %2, i8* %p release, align 1 + ret void +} + +define void @or_16i(i16* %p) { +; X64-LABEL: or_16i: ; X64-NOT: orw -; X32-LABEL: or_16 +; X32-LABEL: or_16i: ; X32-NOT: orw %1 = load atomic i16, i16* %p acquire, align 2 %2 = or i16 %1, 2 @@ -258,12 +454,23 @@ define void @or_16(i16* %p) { ret void } -define void @or_32(i32* %p) { -; X64-LABEL: or_32 +define void @or_16r(i16* %p, i16 %v) { +; X64-LABEL: or_16r: +; X64-NOT: orw +; X32-LABEL: or_16r: +; X32-NOT: orw [.*], ( + %1 = load atomic i16, i16* %p acquire, align 2 + %2 = or i16 %1, %v + store atomic i16 %2, i16* %p release, align 2 + ret void +} + +define void @or_32i(i32* %p) { +; X64-LABEL: or_32i: ; X64-NOT: lock ; X64: orl ; X64-NOT: movl -; X32-LABEL: or_32 +; X32-LABEL: or_32i: ; X32-NOT: lock ; X32: orl ; X32-NOT: movl @@ -273,23 +480,51 @@ define void @or_32(i32* %p) { ret void } -define void @or_64(i64* %p) { -; X64-LABEL: or_64 +define void @or_32r(i32* %p, i32 %v) { +; X64-LABEL: or_32r: +; X64-NOT: lock +; X64: orl +; X64-NOT: movl +; X32-LABEL: or_32r: +; X32-NOT: lock +; X32: orl +; X32-NOT: movl + %1 = load atomic i32, i32* %p acquire, align 4 + %2 = or i32 %1, %v + store atomic i32 %2, i32* %p release, align 4 + ret void +} + +define void @or_64i(i64* %p) { +; X64-LABEL: or_64i: ; X64-NOT: lock ; X64: orq ; X64-NOT: movq ; We do not check X86-32 as it cannot do 'orq'. -; X32-LABEL: or_64 +; X32-LABEL: or_64i: %1 = load atomic i64, i64* %p acquire, align 8 %2 = or i64 %1, 2 store atomic i64 %2, i64* %p release, align 8 ret void } -define void @or_32_seq_cst(i32* %p) { -; X64-LABEL: or_32_seq_cst +define void @or_64r(i64* %p, i64 %v) { +; X64-LABEL: or_64r: +; X64-NOT: lock +; X64: orq +; X64-NOT: movq +; We do not check X86-32 as it cannot do 'orq'. +; X32-LABEL: or_64r: + %1 = load atomic i64, i64* %p acquire, align 8 + %2 = or i64 %1, %v + store atomic i64 %2, i64* %p release, align 8 + ret void +} + +define void @or_32i_seq_cst(i32* %p) { +; X64-LABEL: or_32i_seq_cst: ; X64: xchgl -; X32-LABEL: or_32_seq_cst +; X32-LABEL: or_32i_seq_cst: ; X32: xchgl %1 = load atomic i32, i32* %p monotonic, align 4 %2 = or i32 %1, 2 @@ -297,14 +532,25 @@ define void @or_32_seq_cst(i32* %p) { ret void } +define void @or_32r_seq_cst(i32* %p, i32 %v) { +; X64-LABEL: or_32r_seq_cst: +; X64: xchgl +; X32-LABEL: or_32r_seq_cst: +; X32: xchgl + %1 = load atomic i32, i32* %p monotonic, align 4 + %2 = or i32 %1, %v + store atomic i32 %2, i32* %p seq_cst, align 4 + ret void +} + ; ----- XOR ----- -define void @xor_8(i8* %p) { -; X64-LABEL: xor_8 +define void @xor_8i(i8* %p) { +; X64-LABEL: xor_8i: ; X64-NOT: lock ; X64: xorb ; X64-NOT: movb -; X32-LABEL: xor_8 +; X32-LABEL: xor_8i: ; X32-NOT: lock ; X32: xorb ; X32-NOT: movb @@ -314,10 +560,25 @@ define void @xor_8(i8* %p) { ret void } -define void @xor_16(i16* %p) { -; X64-LABEL: xor_16 +define void @xor_8r(i8* %p, i8 %v) { +; X64-LABEL: xor_8r: +; X64-NOT: lock +; X64: xorb +; X64-NOT: movb +; X32-LABEL: xor_8r: +; X32-NOT: lock +; X32: xorb +; X32-NOT: movb + %1 = load atomic i8, i8* %p acquire, align 1 + %2 = xor i8 %1, %v + store atomic i8 %2, i8* %p release, align 1 + ret void +} + +define void @xor_16i(i16* %p) { +; X64-LABEL: xor_16i: ; X64-NOT: xorw -; X32-LABEL: xor_16 +; X32-LABEL: xor_16i: ; X32-NOT: xorw %1 = load atomic i16, i16* %p acquire, align 2 %2 = xor i16 %1, 2 @@ -325,12 +586,23 @@ define void @xor_16(i16* %p) { ret void } -define void @xor_32(i32* %p) { -; X64-LABEL: xor_32 +define void @xor_16r(i16* %p, i16 %v) { +; X64-LABEL: xor_16r: +; X64-NOT: xorw +; X32-LABEL: xor_16r: +; X32-NOT: xorw [.*], ( + %1 = load atomic i16, i16* %p acquire, align 2 + %2 = xor i16 %1, %v + store atomic i16 %2, i16* %p release, align 2 + ret void +} + +define void @xor_32i(i32* %p) { +; X64-LABEL: xor_32i: ; X64-NOT: lock ; X64: xorl ; X64-NOT: movl -; X32-LABEL: xor_32 +; X32-LABEL: xor_32i: ; X32-NOT: lock ; X32: xorl ; X32-NOT: movl @@ -340,23 +612,51 @@ define void @xor_32(i32* %p) { ret void } -define void @xor_64(i64* %p) { -; X64-LABEL: xor_64 +define void @xor_32r(i32* %p, i32 %v) { +; X64-LABEL: xor_32r: +; X64-NOT: lock +; X64: xorl +; X64-NOT: movl +; X32-LABEL: xor_32r: +; X32-NOT: lock +; X32: xorl +; X32-NOT: movl + %1 = load atomic i32, i32* %p acquire, align 4 + %2 = xor i32 %1, %v + store atomic i32 %2, i32* %p release, align 4 + ret void +} + +define void @xor_64i(i64* %p) { +; X64-LABEL: xor_64i: ; X64-NOT: lock ; X64: xorq ; X64-NOT: movq ; We do not check X86-32 as it cannot do 'xorq'. -; X32-LABEL: xor_64 +; X32-LABEL: xor_64i: %1 = load atomic i64, i64* %p acquire, align 8 %2 = xor i64 %1, 2 store atomic i64 %2, i64* %p release, align 8 ret void } -define void @xor_32_seq_cst(i32* %p) { -; X64-LABEL: xor_32_seq_cst +define void @xor_64r(i64* %p, i64 %v) { +; X64-LABEL: xor_64r: +; X64-NOT: lock +; X64: xorq +; X64-NOT: movq +; We do not check X86-32 as it cannot do 'xorq'. +; X32-LABEL: xor_64r: + %1 = load atomic i64, i64* %p acquire, align 8 + %2 = xor i64 %1, %v + store atomic i64 %2, i64* %p release, align 8 + ret void +} + +define void @xor_32i_seq_cst(i32* %p) { +; X64-LABEL: xor_32i_seq_cst: ; X64: xchgl -; X32-LABEL: xor_32_seq_cst +; X32-LABEL: xor_32i_seq_cst: ; X32: xchgl %1 = load atomic i32, i32* %p monotonic, align 4 %2 = xor i32 %1, 2 @@ -364,18 +664,29 @@ define void @xor_32_seq_cst(i32* %p) { ret void } +define void @xor_32r_seq_cst(i32* %p, i32 %v) { +; X64-LABEL: xor_32r_seq_cst: +; X64: xchgl +; X32-LABEL: xor_32r_seq_cst: +; X32: xchgl + %1 = load atomic i32, i32* %p monotonic, align 4 + %2 = xor i32 %1, %v + store atomic i32 %2, i32* %p seq_cst, align 4 + ret void +} + ; ----- INC ----- define void @inc_8(i8* %p) { -; X64-LABEL: inc_8 +; X64-LABEL: inc_8: ; X64-NOT: lock ; X64: incb ; X64-NOT: movb -; X32-LABEL: inc_8 +; X32-LABEL: inc_8: ; X32-NOT: lock ; X32: incb ; X32-NOT: movb -; SLOW_INC-LABEL: inc_8 +; SLOW_INC-LABEL: inc_8: ; SLOW_INC-NOT: incb ; SLOW_INC-NOT: movb %1 = load atomic i8, i8* %p seq_cst, align 1 @@ -387,11 +698,11 @@ define void @inc_8(i8* %p) { define void @inc_16(i16* %p) { ; Currently the transformation is not done on 16 bit accesses, as the backend ; treat 16 bit arithmetic as expensive on X86/X86_64. -; X64-LABEL: inc_16 +; X64-LABEL: inc_16: ; X64-NOT: incw -; X32-LABEL: inc_16 +; X32-LABEL: inc_16: ; X32-NOT: incw -; SLOW_INC-LABEL: inc_16 +; SLOW_INC-LABEL: inc_16: ; SLOW_INC-NOT: incw %1 = load atomic i16, i16* %p acquire, align 2 %2 = add i16 %1, 1 @@ -400,15 +711,15 @@ define void @inc_16(i16* %p) { } define void @inc_32(i32* %p) { -; X64-LABEL: inc_32 +; X64-LABEL: inc_32: ; X64-NOT: lock ; X64: incl ; X64-NOT: movl -; X32-LABEL: inc_32 +; X32-LABEL: inc_32: ; X32-NOT: lock ; X32: incl ; X32-NOT: movl -; SLOW_INC-LABEL: inc_32 +; SLOW_INC-LABEL: inc_32: ; SLOW_INC-NOT: incl ; SLOW_INC-NOT: movl %1 = load atomic i32, i32* %p acquire, align 4 @@ -418,13 +729,13 @@ define void @inc_32(i32* %p) { } define void @inc_64(i64* %p) { -; X64-LABEL: inc_64 +; X64-LABEL: inc_64: ; X64-NOT: lock ; X64: incq ; X64-NOT: movq ; We do not check X86-32 as it cannot do 'incq'. -; X32-LABEL: inc_64 -; SLOW_INC-LABEL: inc_64 +; X32-LABEL: inc_64: +; SLOW_INC-LABEL: inc_64: ; SLOW_INC-NOT: incq ; SLOW_INC-NOT: movq %1 = load atomic i64, i64* %p acquire, align 8 @@ -434,9 +745,9 @@ define void @inc_64(i64* %p) { } define void @inc_32_seq_cst(i32* %p) { -; X64-LABEL: inc_32_seq_cst +; X64-LABEL: inc_32_seq_cst: ; X64: xchgl -; X32-LABEL: inc_32_seq_cst +; X32-LABEL: inc_32_seq_cst: ; X32: xchgl %1 = load atomic i32, i32* %p monotonic, align 4 %2 = add i32 %1, 1 @@ -447,15 +758,15 @@ define void @inc_32_seq_cst(i32* %p) { ; ----- DEC ----- define void @dec_8(i8* %p) { -; X64-LABEL: dec_8 +; X64-LABEL: dec_8: ; X64-NOT: lock ; X64: decb ; X64-NOT: movb -; X32-LABEL: dec_8 +; X32-LABEL: dec_8: ; X32-NOT: lock ; X32: decb ; X32-NOT: movb -; SLOW_INC-LABEL: dec_8 +; SLOW_INC-LABEL: dec_8: ; SLOW_INC-NOT: decb ; SLOW_INC-NOT: movb %1 = load atomic i8, i8* %p seq_cst, align 1 @@ -467,11 +778,11 @@ define void @dec_8(i8* %p) { define void @dec_16(i16* %p) { ; Currently the transformation is not done on 16 bit accesses, as the backend ; treat 16 bit arithmetic as expensive on X86/X86_64. -; X64-LABEL: dec_16 +; X64-LABEL: dec_16: ; X64-NOT: decw -; X32-LABEL: dec_16 +; X32-LABEL: dec_16: ; X32-NOT: decw -; SLOW_INC-LABEL: dec_16 +; SLOW_INC-LABEL: dec_16: ; SLOW_INC-NOT: decw %1 = load atomic i16, i16* %p acquire, align 2 %2 = sub i16 %1, 1 @@ -480,15 +791,15 @@ define void @dec_16(i16* %p) { } define void @dec_32(i32* %p) { -; X64-LABEL: dec_32 +; X64-LABEL: dec_32: ; X64-NOT: lock ; X64: decl ; X64-NOT: movl -; X32-LABEL: dec_32 +; X32-LABEL: dec_32: ; X32-NOT: lock ; X32: decl ; X32-NOT: movl -; SLOW_INC-LABEL: dec_32 +; SLOW_INC-LABEL: dec_32: ; SLOW_INC-NOT: decl ; SLOW_INC-NOT: movl %1 = load atomic i32, i32* %p acquire, align 4 @@ -498,13 +809,13 @@ define void @dec_32(i32* %p) { } define void @dec_64(i64* %p) { -; X64-LABEL: dec_64 +; X64-LABEL: dec_64: ; X64-NOT: lock ; X64: decq ; X64-NOT: movq ; We do not check X86-32 as it cannot do 'decq'. -; X32-LABEL: dec_64 -; SLOW_INC-LABEL: dec_64 +; X32-LABEL: dec_64: +; SLOW_INC-LABEL: dec_64: ; SLOW_INC-NOT: decq ; SLOW_INC-NOT: movq %1 = load atomic i64, i64* %p acquire, align 8 @@ -514,12 +825,157 @@ define void @dec_64(i64* %p) { } define void @dec_32_seq_cst(i32* %p) { -; X64-LABEL: dec_32_seq_cst +; X64-LABEL: dec_32_seq_cst: ; X64: xchgl -; X32-LABEL: dec_32_seq_cst +; X32-LABEL: dec_32_seq_cst: ; X32: xchgl %1 = load atomic i32, i32* %p monotonic, align 4 %2 = sub i32 %1, 1 store atomic i32 %2, i32* %p seq_cst, align 4 ret void } + +; ----- FADD ----- + +define void @fadd_32r(float* %loc, float %val) { +; X64-LABEL: fadd_32r: +; X64-NOT: lock +; X64-NOT: mov +; X64: addss (%[[M:[a-z]+]]), %[[XMM:xmm[0-9]+]] +; X64-NEXT: movss %[[XMM]], (%[[M]]) +; X32-LABEL: fadd_32r: +; Don't check x86-32. +; LLVM's SSE handling is conservative on x86-32 even without using atomics. + %floc = bitcast float* %loc to i32* + %1 = load atomic i32, i32* %floc seq_cst, align 4 + %2 = bitcast i32 %1 to float + %add = fadd float %2, %val + %3 = bitcast float %add to i32 + store atomic i32 %3, i32* %floc release, align 4 + ret void +} + +define void @fadd_64r(double* %loc, double %val) { +; X64-LABEL: fadd_64r: +; X64-NOT: lock +; X64-NOT: mov +; X64: addsd (%[[M:[a-z]+]]), %[[XMM:xmm[0-9]+]] +; X64-NEXT: movsd %[[XMM]], (%[[M]]) +; X32-LABEL: fadd_64r: +; Don't check x86-32 (see comment above). + %floc = bitcast double* %loc to i64* + %1 = load atomic i64, i64* %floc seq_cst, align 8 + %2 = bitcast i64 %1 to double + %add = fadd double %2, %val + %3 = bitcast double %add to i64 + store atomic i64 %3, i64* %floc release, align 8 + ret void +} + +@glob32 = global float 0.000000e+00, align 4 +@glob64 = global double 0.000000e+00, align 8 + +; Floating-point add to a global using an immediate. +define void @fadd_32g() { +; X64-LABEL: fadd_32g: +; X64-NOT: lock +; X64: movss .{{[A-Z0-9_]+}}(%rip), %[[XMM:xmm[0-9]+]] +; X64-NEXT: addss glob32(%rip), %[[XMM]] +; X64-NEXT: movss %[[XMM]], glob32(%rip) +; X32-LABEL: fadd_32g: +; Don't check x86-32 (see comment above). + %i = load atomic i32, i32* bitcast (float* @glob32 to i32*) monotonic, align 4 + %f = bitcast i32 %i to float + %add = fadd float %f, 1.000000e+00 + %s = bitcast float %add to i32 + store atomic i32 %s, i32* bitcast (float* @glob32 to i32*) monotonic, align 4 + ret void +} + +define void @fadd_64g() { +; X64-LABEL: fadd_64g: +; X64-NOT: lock +; X64: movsd .{{[A-Z0-9_]+}}(%rip), %[[XMM:xmm[0-9]+]] +; X64-NEXT: addsd glob64(%rip), %[[XMM]] +; X64-NEXT: movsd %[[XMM]], glob64(%rip) +; X32-LABEL: fadd_64g: +; Don't check x86-32 (see comment above). + %i = load atomic i64, i64* bitcast (double* @glob64 to i64*) monotonic, align 8 + %f = bitcast i64 %i to double + %add = fadd double %f, 1.000000e+00 + %s = bitcast double %add to i64 + store atomic i64 %s, i64* bitcast (double* @glob64 to i64*) monotonic, align 8 + ret void +} + +; Floating-point add to a hard-coded immediate location using an immediate. +define void @fadd_32imm() { +; X64-LABEL: fadd_32imm: +; X64-NOT: lock +; X64: movl $3735928559, %e[[M:[a-z]+]] +; X64: movss .{{[A-Z0-9_]+}}(%rip), %[[XMM:xmm[0-9]+]] +; X64-NEXT: addss (%r[[M]]), %[[XMM]] +; X64-NEXT: movss %[[XMM]], (%r[[M]]) +; X32-LABEL: fadd_32imm: +; Don't check x86-32 (see comment above). + %i = load atomic i32, i32* inttoptr (i32 3735928559 to i32*) monotonic, align 4 + %f = bitcast i32 %i to float + %add = fadd float %f, 1.000000e+00 + %s = bitcast float %add to i32 + store atomic i32 %s, i32* inttoptr (i32 3735928559 to i32*) monotonic, align 4 + ret void +} + +define void @fadd_64imm() { +; X64-LABEL: fadd_64imm: +; X64-NOT: lock +; X64: movl $3735928559, %e[[M:[a-z]+]] +; X64: movsd .{{[A-Z0-9_]+}}(%rip), %[[XMM:xmm[0-9]+]] +; X64-NEXT: addsd (%r[[M]]), %[[XMM]] +; X64-NEXT: movsd %[[XMM]], (%r[[M]]) +; X32-LABEL: fadd_64imm: +; Don't check x86-32 (see comment above). + %i = load atomic i64, i64* inttoptr (i64 3735928559 to i64*) monotonic, align 8 + %f = bitcast i64 %i to double + %add = fadd double %f, 1.000000e+00 + %s = bitcast double %add to i64 + store atomic i64 %s, i64* inttoptr (i64 3735928559 to i64*) monotonic, align 8 + ret void +} + +; Floating-point add to a stack location. +define void @fadd_32stack() { +; X64-LABEL: fadd_32stack: +; X64-NOT: lock +; X64: movss .{{[A-Z0-9_]+}}(%rip), %[[XMM:xmm[0-9]+]] +; X64-NEXT: addss [[STACKOFF:-?[0-9]+]](%rsp), %[[XMM]] +; X64-NEXT: movss %[[XMM]], [[STACKOFF]](%rsp) +; X32-LABEL: fadd_32stack: +; Don't check x86-32 (see comment above). + %ptr = alloca i32, align 4 + %bc3 = bitcast i32* %ptr to float* + %load = load atomic i32, i32* %ptr acquire, align 4 + %bc0 = bitcast i32 %load to float + %fadd = fadd float 1.000000e+00, %bc0 + %bc1 = bitcast float %fadd to i32 + store atomic i32 %bc1, i32* %ptr release, align 4 + ret void +} + +define void @fadd_64stack() { +; X64-LABEL: fadd_64stack: +; X64-NOT: lock +; X64: movsd .{{[A-Z0-9_]+}}(%rip), %[[XMM:xmm[0-9]+]] +; X64-NEXT: addsd [[STACKOFF:-?[0-9]+]](%rsp), %[[XMM]] +; X64-NEXT: movsd %[[XMM]], [[STACKOFF]](%rsp) +; X32-LABEL: fadd_64stack: +; Don't check x86-32 (see comment above). + %ptr = alloca i64, align 8 + %bc3 = bitcast i64* %ptr to double* + %load = load atomic i64, i64* %ptr acquire, align 8 + %bc0 = bitcast i64 %load to double + %fadd = fadd double 1.000000e+00, %bc0 + %bc1 = bitcast double %fadd to i64 + store atomic i64 %bc1, i64* %ptr release, align 8 + ret void +} diff --git a/test/CodeGen/X86/avg.ll b/test/CodeGen/X86/avg.ll new file mode 100644 index 000000000000..f1c636a73305 --- /dev/null +++ b/test/CodeGen/X86/avg.ll @@ -0,0 +1,724 @@ +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse2 | FileCheck %s --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512BW + +define void @avg_v4i8(<4 x i8>* %a, <4 x i8>* %b) { +; SSE2-LABEL: avg_v4i8: +; SSE2: # BB#0: +; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero +; SSE2-NEXT: pavgb %xmm0, %xmm1 +; SSE2-NEXT: movd %xmm1, (%rax) +; SSE2-NEXT: retq +; +; AVX2-LABEL: avg_v4i8: +; AVX2: # BB#0: +; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; AVX2-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero +; AVX2-NEXT: vpavgb %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vmovd %xmm0, (%rax) +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: avg_v4i8: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovd (%rdi), %xmm0 +; AVX512BW-NEXT: vmovd (%rsi), %xmm1 +; AVX512BW-NEXT: vpavgb %xmm0, %xmm1, %xmm0 +; AVX512BW-NEXT: vmovd %xmm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <4 x i8>, <4 x i8>* %a + %2 = load <4 x i8>, <4 x i8>* %b + %3 = zext <4 x i8> %1 to <4 x i32> + %4 = zext <4 x i8> %2 to <4 x i32> + %5 = add nuw nsw <4 x i32> %3, + %6 = add nuw nsw <4 x i32> %5, %4 + %7 = lshr <4 x i32> %6, + %8 = trunc <4 x i32> %7 to <4 x i8> + store <4 x i8> %8, <4 x i8>* undef, align 4 + ret void +} + +define void @avg_v8i8(<8 x i8>* %a, <8 x i8>* %b) { +; SSE2-LABEL: avg_v8i8: +; SSE2: # BB#0: +; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero +; SSE2-NEXT: pavgb %xmm0, %xmm1 +; SSE2-NEXT: movq %xmm1, (%rax) +; SSE2-NEXT: retq +; +; AVX2-LABEL: avg_v8i8: +; AVX2: # BB#0: +; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX2-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX2-NEXT: vpavgb %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vmovq %xmm0, (%rax) +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: avg_v8i8: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovq (%rdi), %xmm0 +; AVX512BW-NEXT: vmovq (%rsi), %xmm1 +; AVX512BW-NEXT: vpavgb %xmm0, %xmm1, %xmm0 +; AVX512BW-NEXT: vmovq %xmm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <8 x i8>, <8 x i8>* %a + %2 = load <8 x i8>, <8 x i8>* %b + %3 = zext <8 x i8> %1 to <8 x i32> + %4 = zext <8 x i8> %2 to <8 x i32> + %5 = add nuw nsw <8 x i32> %3, + %6 = add nuw nsw <8 x i32> %5, %4 + %7 = lshr <8 x i32> %6, + %8 = trunc <8 x i32> %7 to <8 x i8> + store <8 x i8> %8, <8 x i8>* undef, align 4 + ret void +} + +define void @avg_v16i8(<16 x i8>* %a, <16 x i8>* %b) { +; SSE2-LABEL: avg_v16i8: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa (%rsi), %xmm0 +; SSE2-NEXT: pavgb (%rdi), %xmm0 +; SSE2-NEXT: movdqu %xmm0, (%rax) +; SSE2-NEXT: retq +; +; AVX-LABEL: avg_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa (%rsi), %xmm0 +; AVX-NEXT: vpavgb (%rdi), %xmm0, %xmm0 +; AVX-NEXT: vmovdqu %xmm0, (%rax) +; AVX-NEXT: retq + %1 = load <16 x i8>, <16 x i8>* %a + %2 = load <16 x i8>, <16 x i8>* %b + %3 = zext <16 x i8> %1 to <16 x i32> + %4 = zext <16 x i8> %2 to <16 x i32> + %5 = add nuw nsw <16 x i32> %3, + %6 = add nuw nsw <16 x i32> %5, %4 + %7 = lshr <16 x i32> %6, + %8 = trunc <16 x i32> %7 to <16 x i8> + store <16 x i8> %8, <16 x i8>* undef, align 4 + ret void +} + +define void @avg_v32i8(<32 x i8>* %a, <32 x i8>* %b) { +; AVX2-LABEL: avg_v32i8: +; AVX2: # BB#0: +; AVX2-NEXT: vmovdqa (%rsi), %ymm0 +; AVX2-NEXT: vpavgb (%rdi), %ymm0, %ymm0 +; AVX2-NEXT: vmovdqu %ymm0, (%rax) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: avg_v32i8: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovdqa (%rsi), %ymm0 +; AVX512BW-NEXT: vpavgb (%rdi), %ymm0, %ymm0 +; AVX512BW-NEXT: vmovdqu %ymm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <32 x i8>, <32 x i8>* %a + %2 = load <32 x i8>, <32 x i8>* %b + %3 = zext <32 x i8> %1 to <32 x i32> + %4 = zext <32 x i8> %2 to <32 x i32> + %5 = add nuw nsw <32 x i32> %3, + %6 = add nuw nsw <32 x i32> %5, %4 + %7 = lshr <32 x i32> %6, + %8 = trunc <32 x i32> %7 to <32 x i8> + store <32 x i8> %8, <32 x i8>* undef, align 4 + ret void +} + +define void @avg_v64i8(<64 x i8>* %a, <64 x i8>* %b) { +; AVX512BW-LABEL: avg_v64i8: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovdqu8 (%rsi), %zmm0 +; AVX512BW-NEXT: vpavgb (%rdi), %zmm0, %zmm0 +; AVX512BW-NEXT: vmovdqu8 %zmm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <64 x i8>, <64 x i8>* %a + %2 = load <64 x i8>, <64 x i8>* %b + %3 = zext <64 x i8> %1 to <64 x i32> + %4 = zext <64 x i8> %2 to <64 x i32> + %5 = add nuw nsw <64 x i32> %3, + %6 = add nuw nsw <64 x i32> %5, %4 + %7 = lshr <64 x i32> %6, + %8 = trunc <64 x i32> %7 to <64 x i8> + store <64 x i8> %8, <64 x i8>* undef, align 4 + ret void +} + +define void @avg_v4i16(<4 x i16>* %a, <4 x i16>* %b) { +; SSE2-LABEL: avg_v4i16: +; SSE2: # BB#0: +; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero +; SSE2-NEXT: pavgw %xmm0, %xmm1 +; SSE2-NEXT: movq %xmm1, (%rax) +; SSE2-NEXT: retq +; +; AVX2-LABEL: avg_v4i16: +; AVX2: # BB#0: +; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX2-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX2-NEXT: vpavgw %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vmovq %xmm0, (%rax) +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: avg_v4i16: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovq (%rdi), %xmm0 +; AVX512BW-NEXT: vmovq (%rsi), %xmm1 +; AVX512BW-NEXT: vpavgw %xmm0, %xmm1, %xmm0 +; AVX512BW-NEXT: vmovq %xmm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <4 x i16>, <4 x i16>* %a + %2 = load <4 x i16>, <4 x i16>* %b + %3 = zext <4 x i16> %1 to <4 x i32> + %4 = zext <4 x i16> %2 to <4 x i32> + %5 = add nuw nsw <4 x i32> %3, + %6 = add nuw nsw <4 x i32> %5, %4 + %7 = lshr <4 x i32> %6, + %8 = trunc <4 x i32> %7 to <4 x i16> + store <4 x i16> %8, <4 x i16>* undef, align 4 + ret void +} + +define void @avg_v8i16(<8 x i16>* %a, <8 x i16>* %b) { +; SSE2-LABEL: avg_v8i16: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa (%rsi), %xmm0 +; SSE2-NEXT: pavgw (%rdi), %xmm0 +; SSE2-NEXT: movdqu %xmm0, (%rax) +; SSE2-NEXT: retq +; +; AVX-LABEL: avg_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa (%rsi), %xmm0 +; AVX-NEXT: vpavgw (%rdi), %xmm0, %xmm0 +; AVX-NEXT: vmovdqu %xmm0, (%rax) +; AVX-NEXT: retq + %1 = load <8 x i16>, <8 x i16>* %a + %2 = load <8 x i16>, <8 x i16>* %b + %3 = zext <8 x i16> %1 to <8 x i32> + %4 = zext <8 x i16> %2 to <8 x i32> + %5 = add nuw nsw <8 x i32> %3, + %6 = add nuw nsw <8 x i32> %5, %4 + %7 = lshr <8 x i32> %6, + %8 = trunc <8 x i32> %7 to <8 x i16> + store <8 x i16> %8, <8 x i16>* undef, align 4 + ret void +} + +define void @avg_v16i16(<16 x i16>* %a, <16 x i16>* %b) { +; AVX2-LABEL: avg_v16i16: +; AVX2: # BB#0: +; AVX2-NEXT: vmovdqa (%rsi), %ymm0 +; AVX2-NEXT: vpavgw (%rdi), %ymm0, %ymm0 +; AVX2-NEXT: vmovdqu %ymm0, (%rax) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: avg_v16i16: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovdqa (%rsi), %ymm0 +; AVX512BW-NEXT: vpavgw (%rdi), %ymm0, %ymm0 +; AVX512BW-NEXT: vmovdqu %ymm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <16 x i16>, <16 x i16>* %a + %2 = load <16 x i16>, <16 x i16>* %b + %3 = zext <16 x i16> %1 to <16 x i32> + %4 = zext <16 x i16> %2 to <16 x i32> + %5 = add nuw nsw <16 x i32> %3, + %6 = add nuw nsw <16 x i32> %5, %4 + %7 = lshr <16 x i32> %6, + %8 = trunc <16 x i32> %7 to <16 x i16> + store <16 x i16> %8, <16 x i16>* undef, align 4 + ret void +} + +define void @avg_v32i16(<32 x i16>* %a, <32 x i16>* %b) { +; AVX512BW-LABEL: avg_v32i16: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovdqu16 (%rsi), %zmm0 +; AVX512BW-NEXT: vpavgw (%rdi), %zmm0, %zmm0 +; AVX512BW-NEXT: vmovdqu16 %zmm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <32 x i16>, <32 x i16>* %a + %2 = load <32 x i16>, <32 x i16>* %b + %3 = zext <32 x i16> %1 to <32 x i32> + %4 = zext <32 x i16> %2 to <32 x i32> + %5 = add nuw nsw <32 x i32> %3, + %6 = add nuw nsw <32 x i32> %5, %4 + %7 = lshr <32 x i32> %6, + %8 = trunc <32 x i32> %7 to <32 x i16> + store <32 x i16> %8, <32 x i16>* undef, align 4 + ret void +} + +define void @avg_v4i8_2(<4 x i8>* %a, <4 x i8>* %b) { +; SSE2-LABEL: avg_v4i8_2: +; SSE2: # BB#0: +; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero +; SSE2-NEXT: pavgb %xmm0, %xmm1 +; SSE2-NEXT: movd %xmm1, (%rax) +; SSE2-NEXT: retq +; +; AVX2-LABEL: avg_v4i8_2: +; AVX2: # BB#0: +; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; AVX2-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero +; AVX2-NEXT: vpavgb %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vmovd %xmm0, (%rax) +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: avg_v4i8_2: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovd (%rdi), %xmm0 +; AVX512BW-NEXT: vmovd (%rsi), %xmm1 +; AVX512BW-NEXT: vpavgb %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: vmovd %xmm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <4 x i8>, <4 x i8>* %a + %2 = load <4 x i8>, <4 x i8>* %b + %3 = zext <4 x i8> %1 to <4 x i32> + %4 = zext <4 x i8> %2 to <4 x i32> + %5 = add nuw nsw <4 x i32> %3, %4 + %6 = add nuw nsw <4 x i32> %5, + %7 = lshr <4 x i32> %6, + %8 = trunc <4 x i32> %7 to <4 x i8> + store <4 x i8> %8, <4 x i8>* undef, align 4 + ret void +} + +define void @avg_v8i8_2(<8 x i8>* %a, <8 x i8>* %b) { +; SSE2-LABEL: avg_v8i8_2: +; SSE2: # BB#0: +; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero +; SSE2-NEXT: pavgb %xmm0, %xmm1 +; SSE2-NEXT: movq %xmm1, (%rax) +; SSE2-NEXT: retq +; +; AVX2-LABEL: avg_v8i8_2: +; AVX2: # BB#0: +; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX2-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX2-NEXT: vpavgb %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vmovq %xmm0, (%rax) +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: avg_v8i8_2: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovq (%rdi), %xmm0 +; AVX512BW-NEXT: vmovq (%rsi), %xmm1 +; AVX512BW-NEXT: vpavgb %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: vmovq %xmm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <8 x i8>, <8 x i8>* %a + %2 = load <8 x i8>, <8 x i8>* %b + %3 = zext <8 x i8> %1 to <8 x i32> + %4 = zext <8 x i8> %2 to <8 x i32> + %5 = add nuw nsw <8 x i32> %3, %4 + %6 = add nuw nsw <8 x i32> %5, + %7 = lshr <8 x i32> %6, + %8 = trunc <8 x i32> %7 to <8 x i8> + store <8 x i8> %8, <8 x i8>* undef, align 4 + ret void +} + +define void @avg_v16i8_2(<16 x i8>* %a, <16 x i8>* %b) { +; SSE2-LABEL: avg_v16i8_2: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa (%rdi), %xmm0 +; SSE2-NEXT: pavgb (%rsi), %xmm0 +; SSE2-NEXT: movdqu %xmm0, (%rax) +; SSE2-NEXT: retq +; +; AVX-LABEL: avg_v16i8_2: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa (%rdi), %xmm0 +; AVX-NEXT: vpavgb (%rsi), %xmm0, %xmm0 +; AVX-NEXT: vmovdqu %xmm0, (%rax) +; AVX-NEXT: retq + %1 = load <16 x i8>, <16 x i8>* %a + %2 = load <16 x i8>, <16 x i8>* %b + %3 = zext <16 x i8> %1 to <16 x i32> + %4 = zext <16 x i8> %2 to <16 x i32> + %5 = add nuw nsw <16 x i32> %3, %4 + %6 = add nuw nsw <16 x i32> %5, + %7 = lshr <16 x i32> %6, + %8 = trunc <16 x i32> %7 to <16 x i8> + store <16 x i8> %8, <16 x i8>* undef, align 4 + ret void +} + +define void @avg_v32i8_2(<32 x i8>* %a, <32 x i8>* %b) { +; AVX2-LABEL: avg_v32i8_2: +; AVX2: # BB#0: +; AVX2-NEXT: vmovdqa (%rdi), %ymm0 +; AVX2-NEXT: vpavgb (%rsi), %ymm0, %ymm0 +; AVX2-NEXT: vmovdqu %ymm0, (%rax) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: avg_v32i8_2: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0 +; AVX512BW-NEXT: vpavgb (%rsi), %ymm0, %ymm0 +; AVX512BW-NEXT: vmovdqu %ymm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <32 x i8>, <32 x i8>* %a + %2 = load <32 x i8>, <32 x i8>* %b + %3 = zext <32 x i8> %1 to <32 x i32> + %4 = zext <32 x i8> %2 to <32 x i32> + %5 = add nuw nsw <32 x i32> %3, %4 + %6 = add nuw nsw <32 x i32> %5, + %7 = lshr <32 x i32> %6, + %8 = trunc <32 x i32> %7 to <32 x i8> + store <32 x i8> %8, <32 x i8>* undef, align 4 + ret void +} + +define void @avg_v64i8_2(<64 x i8>* %a, <64 x i8>* %b) { +; AVX512BW-LABEL: avg_v64i8_2: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovdqu8 (%rsi), %zmm0 +; AVX512BW-NEXT: vpavgb %zmm0, %zmm0, %zmm0 +; AVX512BW-NEXT: vmovdqu8 %zmm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <64 x i8>, <64 x i8>* %a + %2 = load <64 x i8>, <64 x i8>* %b + %3 = zext <64 x i8> %1 to <64 x i32> + %4 = zext <64 x i8> %2 to <64 x i32> + %5 = add nuw nsw <64 x i32> %4, %4 + %6 = add nuw nsw <64 x i32> %5, + %7 = lshr <64 x i32> %6, + %8 = trunc <64 x i32> %7 to <64 x i8> + store <64 x i8> %8, <64 x i8>* undef, align 4 + ret void +} + + +define void @avg_v4i16_2(<4 x i16>* %a, <4 x i16>* %b) { +; SSE2-LABEL: avg_v4i16_2: +; SSE2: # BB#0: +; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero +; SSE2-NEXT: pavgw %xmm0, %xmm1 +; SSE2-NEXT: movq %xmm1, (%rax) +; SSE2-NEXT: retq +; +; AVX2-LABEL: avg_v4i16_2: +; AVX2: # BB#0: +; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX2-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX2-NEXT: vpavgw %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vmovq %xmm0, (%rax) +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: avg_v4i16_2: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovq (%rdi), %xmm0 +; AVX512BW-NEXT: vmovq (%rsi), %xmm1 +; AVX512BW-NEXT: vpavgw %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: vmovq %xmm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <4 x i16>, <4 x i16>* %a + %2 = load <4 x i16>, <4 x i16>* %b + %3 = zext <4 x i16> %1 to <4 x i32> + %4 = zext <4 x i16> %2 to <4 x i32> + %5 = add nuw nsw <4 x i32> %3, %4 + %6 = add nuw nsw <4 x i32> %5, + %7 = lshr <4 x i32> %6, + %8 = trunc <4 x i32> %7 to <4 x i16> + store <4 x i16> %8, <4 x i16>* undef, align 4 + ret void +} + +define void @avg_v8i16_2(<8 x i16>* %a, <8 x i16>* %b) { +; SSE2-LABEL: avg_v8i16_2: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa (%rdi), %xmm0 +; SSE2-NEXT: pavgw (%rsi), %xmm0 +; SSE2-NEXT: movdqu %xmm0, (%rax) +; SSE2-NEXT: retq +; +; AVX-LABEL: avg_v8i16_2: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa (%rdi), %xmm0 +; AVX-NEXT: vpavgw (%rsi), %xmm0, %xmm0 +; AVX-NEXT: vmovdqu %xmm0, (%rax) +; AVX-NEXT: retq + %1 = load <8 x i16>, <8 x i16>* %a + %2 = load <8 x i16>, <8 x i16>* %b + %3 = zext <8 x i16> %1 to <8 x i32> + %4 = zext <8 x i16> %2 to <8 x i32> + %5 = add nuw nsw <8 x i32> %3, %4 + %6 = add nuw nsw <8 x i32> %5, + %7 = lshr <8 x i32> %6, + %8 = trunc <8 x i32> %7 to <8 x i16> + store <8 x i16> %8, <8 x i16>* undef, align 4 + ret void +} + +define void @avg_v16i16_2(<16 x i16>* %a, <16 x i16>* %b) { +; AVX2-LABEL: avg_v16i16_2: +; AVX2: # BB#0: +; AVX2-NEXT: vmovdqa (%rdi), %ymm0 +; AVX2-NEXT: vpavgw (%rsi), %ymm0, %ymm0 +; AVX2-NEXT: vmovdqu %ymm0, (%rax) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: avg_v16i16_2: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0 +; AVX512BW-NEXT: vpavgw (%rsi), %ymm0, %ymm0 +; AVX512BW-NEXT: vmovdqu %ymm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <16 x i16>, <16 x i16>* %a + %2 = load <16 x i16>, <16 x i16>* %b + %3 = zext <16 x i16> %1 to <16 x i32> + %4 = zext <16 x i16> %2 to <16 x i32> + %5 = add nuw nsw <16 x i32> %3, %4 + %6 = add nuw nsw <16 x i32> %5, + %7 = lshr <16 x i32> %6, + %8 = trunc <16 x i32> %7 to <16 x i16> + store <16 x i16> %8, <16 x i16>* undef, align 4 + ret void +} + +define void @avg_v32i16_2(<32 x i16>* %a, <32 x i16>* %b) { +; AVX512BW-LABEL: avg_v32i16_2: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovdqu16 (%rdi), %zmm0 +; AVX512BW-NEXT: vpavgw (%rsi), %zmm0, %zmm0 +; AVX512BW-NEXT: vmovdqu16 %zmm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <32 x i16>, <32 x i16>* %a + %2 = load <32 x i16>, <32 x i16>* %b + %3 = zext <32 x i16> %1 to <32 x i32> + %4 = zext <32 x i16> %2 to <32 x i32> + %5 = add nuw nsw <32 x i32> %3, %4 + %6 = add nuw nsw <32 x i32> %5, + %7 = lshr <32 x i32> %6, + %8 = trunc <32 x i32> %7 to <32 x i16> + store <32 x i16> %8, <32 x i16>* undef, align 4 + ret void +} + +define void @avg_v4i8_const(<4 x i8>* %a) { +; SSE2-LABEL: avg_v4i8_const: +; SSE2: # BB#0: +; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE2-NEXT: pavgb {{.*}}(%rip), %xmm0 +; SSE2-NEXT: movd %xmm0, (%rax) +; SSE2-NEXT: retq +; +; AVX2-LABEL: avg_v4i8_const: +; AVX2: # BB#0: +; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; AVX2-NEXT: vpavgb {{.*}}(%rip), %xmm0, %xmm0 +; AVX2-NEXT: vmovd %xmm0, (%rax) +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: avg_v4i8_const: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovd (%rdi), %xmm0 +; AVX512BW-NEXT: vpavgb {{.*}}(%rip), %xmm0, %xmm0 +; AVX512BW-NEXT: vmovd %xmm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <4 x i8>, <4 x i8>* %a + %2 = zext <4 x i8> %1 to <4 x i32> + %3 = add nuw nsw <4 x i32> %2, + %4 = lshr <4 x i32> %3, + %5 = trunc <4 x i32> %4 to <4 x i8> + store <4 x i8> %5, <4 x i8>* undef, align 4 + ret void +} + +define void @avg_v8i8_const(<8 x i8>* %a) { +; SSE2-LABEL: avg_v8i8_const: +; SSE2: # BB#0: +; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSE2-NEXT: pavgb {{.*}}(%rip), %xmm0 +; SSE2-NEXT: movq %xmm0, (%rax) +; SSE2-NEXT: retq +; +; AVX2-LABEL: avg_v8i8_const: +; AVX2: # BB#0: +; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX2-NEXT: vpavgb {{.*}}(%rip), %xmm0, %xmm0 +; AVX2-NEXT: vmovq %xmm0, (%rax) +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: avg_v8i8_const: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovq (%rdi), %xmm0 +; AVX512BW-NEXT: vpavgb {{.*}}(%rip), %xmm0, %xmm0 +; AVX512BW-NEXT: vmovq %xmm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <8 x i8>, <8 x i8>* %a + %2 = zext <8 x i8> %1 to <8 x i32> + %3 = add nuw nsw <8 x i32> %2, + %4 = lshr <8 x i32> %3, + %5 = trunc <8 x i32> %4 to <8 x i8> + store <8 x i8> %5, <8 x i8>* undef, align 4 + ret void +} + +define void @avg_v16i8_const(<16 x i8>* %a) { +; SSE2-LABEL: avg_v16i8_const: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa (%rdi), %xmm0 +; SSE2-NEXT: pavgb {{.*}}(%rip), %xmm0 +; SSE2-NEXT: movdqu %xmm0, (%rax) +; SSE2-NEXT: retq +; +; AVX-LABEL: avg_v16i8_const: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa (%rdi), %xmm0 +; AVX-NEXT: vpavgb {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vmovdqu %xmm0, (%rax) +; AVX-NEXT: retq + %1 = load <16 x i8>, <16 x i8>* %a + %2 = zext <16 x i8> %1 to <16 x i32> + %3 = add nuw nsw <16 x i32> %2, + %4 = lshr <16 x i32> %3, + %5 = trunc <16 x i32> %4 to <16 x i8> + store <16 x i8> %5, <16 x i8>* undef, align 4 + ret void +} + +define void @avg_v32i8_const(<32 x i8>* %a) { +; AVX2-LABEL: avg_v32i8_const: +; AVX2: # BB#0: +; AVX2-NEXT: vmovdqa (%rdi), %ymm0 +; AVX2-NEXT: vpavgb {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vmovdqu %ymm0, (%rax) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: avg_v32i8_const: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0 +; AVX512BW-NEXT: vpavgb {{.*}}(%rip), %ymm0, %ymm0 +; AVX512BW-NEXT: vmovdqu %ymm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <32 x i8>, <32 x i8>* %a + %2 = zext <32 x i8> %1 to <32 x i32> + %3 = add nuw nsw <32 x i32> %2, + %4 = lshr <32 x i32> %3, + %5 = trunc <32 x i32> %4 to <32 x i8> + store <32 x i8> %5, <32 x i8>* undef, align 4 + ret void +} + +define void @avg_v64i8_const(<64 x i8>* %a) { +; AVX512BW-LABEL: avg_v64i8_const: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovdqu8 (%rdi), %zmm0 +; AVX512BW-NEXT: vpavgb {{.*}}(%rip), %zmm0, %zmm0 +; AVX512BW-NEXT: vmovdqu8 %zmm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <64 x i8>, <64 x i8>* %a + %2 = zext <64 x i8> %1 to <64 x i32> + %3 = add nuw nsw <64 x i32> %2, + %4 = lshr <64 x i32> %3, + %5 = trunc <64 x i32> %4 to <64 x i8> + store <64 x i8> %5, <64 x i8>* undef, align 4 + ret void +} + +define void @avg_v4i16_const(<4 x i16>* %a) { +; SSE2-LABEL: avg_v4i16_const: +; SSE2: # BB#0: +; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSE2-NEXT: pavgw {{.*}}(%rip), %xmm0 +; SSE2-NEXT: movq %xmm0, (%rax) +; SSE2-NEXT: retq +; +; AVX2-LABEL: avg_v4i16_const: +; AVX2: # BB#0: +; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX2-NEXT: vpavgw {{.*}}(%rip), %xmm0, %xmm0 +; AVX2-NEXT: vmovq %xmm0, (%rax) +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: avg_v4i16_const: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovq (%rdi), %xmm0 +; AVX512BW-NEXT: vpavgw {{.*}}(%rip), %xmm0, %xmm0 +; AVX512BW-NEXT: vmovq %xmm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <4 x i16>, <4 x i16>* %a + %2 = zext <4 x i16> %1 to <4 x i32> + %3 = add nuw nsw <4 x i32> %2, + %4 = lshr <4 x i32> %3, + %5 = trunc <4 x i32> %4 to <4 x i16> + store <4 x i16> %5, <4 x i16>* undef, align 4 + ret void +} + +define void @avg_v8i16_const(<8 x i16>* %a) { +; SSE2-LABEL: avg_v8i16_const: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa (%rdi), %xmm0 +; SSE2-NEXT: pavgw {{.*}}(%rip), %xmm0 +; SSE2-NEXT: movdqu %xmm0, (%rax) +; SSE2-NEXT: retq +; +; AVX-LABEL: avg_v8i16_const: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa (%rdi), %xmm0 +; AVX-NEXT: vpavgw {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vmovdqu %xmm0, (%rax) +; AVX-NEXT: retq + %1 = load <8 x i16>, <8 x i16>* %a + %2 = zext <8 x i16> %1 to <8 x i32> + %3 = add nuw nsw <8 x i32> %2, + %4 = lshr <8 x i32> %3, + %5 = trunc <8 x i32> %4 to <8 x i16> + store <8 x i16> %5, <8 x i16>* undef, align 4 + ret void +} + +define void @avg_v16i16_const(<16 x i16>* %a) { +; AVX2-LABEL: avg_v16i16_const: +; AVX2: # BB#0: +; AVX2-NEXT: vmovdqa (%rdi), %ymm0 +; AVX2-NEXT: vpavgw {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vmovdqu %ymm0, (%rax) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: avg_v16i16_const: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0 +; AVX512BW-NEXT: vpavgw {{.*}}(%rip), %ymm0, %ymm0 +; AVX512BW-NEXT: vmovdqu %ymm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <16 x i16>, <16 x i16>* %a + %2 = zext <16 x i16> %1 to <16 x i32> + %3 = add nuw nsw <16 x i32> %2, + %4 = lshr <16 x i32> %3, + %5 = trunc <16 x i32> %4 to <16 x i16> + store <16 x i16> %5, <16 x i16>* undef, align 4 + ret void +} + +define void @avg_v32i16_const(<32 x i16>* %a) { +; AVX512BW-LABEL: avg_v32i16_const: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vmovdqu16 (%rdi), %zmm0 +; AVX512BW-NEXT: vpavgw {{.*}}(%rip), %zmm0, %zmm0 +; AVX512BW-NEXT: vmovdqu16 %zmm0, (%rax) +; AVX512BW-NEXT: retq + %1 = load <32 x i16>, <32 x i16>* %a + %2 = zext <32 x i16> %1 to <32 x i32> + %3 = add nuw nsw <32 x i32> %2, + %4 = lshr <32 x i32> %3, + %5 = trunc <32 x i32> %4 to <32 x i16> + store <32 x i16> %5, <32 x i16>* undef, align 4 + ret void +} diff --git a/test/CodeGen/X86/avx-cvt-2.ll b/test/CodeGen/X86/avx-cvt-2.ll index 583c7d5947bf..c849312f2367 100644 --- a/test/CodeGen/X86/avx-cvt-2.ll +++ b/test/CodeGen/X86/avx-cvt-2.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx | FileCheck %s ; Check that we generate vector conversion from float to narrower int types diff --git a/test/CodeGen/X86/avx-cvt.ll b/test/CodeGen/X86/avx-cvt.ll index 6df3e5324c11..27339898efdb 100644 --- a/test/CodeGen/X86/avx-cvt.ll +++ b/test/CodeGen/X86/avx-cvt.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s define <8 x float> @sitofp00(<8 x i32> %a) nounwind { @@ -113,8 +114,7 @@ define float @funcD(i64* nocapture %e) nounwind uwtable readonly ssp { define void @fpext() nounwind uwtable { ; CHECK-LABEL: fpext: ; CHECK: # BB#0: -; CHECK-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero -; CHECK-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0 +; CHECK-NEXT: vcvtss2sd -{{[0-9]+}}(%rsp), %xmm0, %xmm0 ; CHECK-NEXT: vmovsd %xmm0, -{{[0-9]+}}(%rsp) ; CHECK-NEXT: retq %f = alloca float, align 4 @@ -138,7 +138,7 @@ declare double @llvm.nearbyint.f64(double %p) define float @floor_f32(float %a) { ; CHECK-LABEL: floor_f32: ; CHECK: # BB#0: -; CHECK-NEXT: vroundss $1, %xmm0, %xmm0, %xmm0 +; CHECK-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0 ; CHECK-NEXT: retq %res = call float @llvm.floor.f32(float %a) ret float %res diff --git a/test/CodeGen/X86/avx-intrinsics-x86-upgrade.ll b/test/CodeGen/X86/avx-intrinsics-x86-upgrade.ll index e2f690bff232..4867869863e3 100644 --- a/test/CodeGen/X86/avx-intrinsics-x86-upgrade.ll +++ b/test/CodeGen/X86/avx-intrinsics-x86-upgrade.ll @@ -143,3 +143,69 @@ define <8 x i16> @test_x86_sse41_pblendw(<8 x i16> %a0, <8 x i16> %a1) { ret <8 x i16> %res } declare <8 x i16> @llvm.x86.sse41.pblendw(<8 x i16>, <8 x i16>, i8) nounwind readnone + + +define <4 x i32> @test_x86_sse41_pmovsxbd(<16 x i8> %a0) { +; CHECK-LABEL: test_x86_sse41_pmovsxbd: +; CHECK: # BB#0: +; CHECK-NEXT: vpmovsxbd %xmm0, %xmm0 +; CHECK-NEXT: retl + %res = call <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8> %a0) ; <<4 x i32>> [#uses=1] + ret <4 x i32> %res +} +declare <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8>) nounwind readnone + + +define <2 x i64> @test_x86_sse41_pmovsxbq(<16 x i8> %a0) { +; CHECK-LABEL: test_x86_sse41_pmovsxbq: +; CHECK: # BB#0: +; CHECK-NEXT: vpmovsxbq %xmm0, %xmm0 +; CHECK-NEXT: retl + %res = call <2 x i64> @llvm.x86.sse41.pmovsxbq(<16 x i8> %a0) ; <<2 x i64>> [#uses=1] + ret <2 x i64> %res +} +declare <2 x i64> @llvm.x86.sse41.pmovsxbq(<16 x i8>) nounwind readnone + + +define <8 x i16> @test_x86_sse41_pmovsxbw(<16 x i8> %a0) { +; CHECK-LABEL: test_x86_sse41_pmovsxbw: +; CHECK: # BB#0: +; CHECK-NEXT: vpmovsxbw %xmm0, %xmm0 +; CHECK-NEXT: retl + %res = call <8 x i16> @llvm.x86.sse41.pmovsxbw(<16 x i8> %a0) ; <<8 x i16>> [#uses=1] + ret <8 x i16> %res +} +declare <8 x i16> @llvm.x86.sse41.pmovsxbw(<16 x i8>) nounwind readnone + + +define <2 x i64> @test_x86_sse41_pmovsxdq(<4 x i32> %a0) { +; CHECK-LABEL: test_x86_sse41_pmovsxdq: +; CHECK: # BB#0: +; CHECK-NEXT: vpmovsxdq %xmm0, %xmm0 +; CHECK-NEXT: retl + %res = call <2 x i64> @llvm.x86.sse41.pmovsxdq(<4 x i32> %a0) ; <<2 x i64>> [#uses=1] + ret <2 x i64> %res +} +declare <2 x i64> @llvm.x86.sse41.pmovsxdq(<4 x i32>) nounwind readnone + + +define <4 x i32> @test_x86_sse41_pmovsxwd(<8 x i16> %a0) { +; CHECK-LABEL: test_x86_sse41_pmovsxwd: +; CHECK: # BB#0: +; CHECK-NEXT: vpmovsxwd %xmm0, %xmm0 +; CHECK-NEXT: retl + %res = call <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16> %a0) ; <<4 x i32>> [#uses=1] + ret <4 x i32> %res +} +declare <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16>) nounwind readnone + + +define <2 x i64> @test_x86_sse41_pmovsxwq(<8 x i16> %a0) { +; CHECK-LABEL: test_x86_sse41_pmovsxwq: +; CHECK: # BB#0: +; CHECK-NEXT: vpmovsxwq %xmm0, %xmm0 +; CHECK-NEXT: retl + %res = call <2 x i64> @llvm.x86.sse41.pmovsxwq(<8 x i16> %a0) ; <<2 x i64>> [#uses=1] + ret <2 x i64> %res +} +declare <2 x i64> @llvm.x86.sse41.pmovsxwq(<8 x i16>) nounwind readnone diff --git a/test/CodeGen/X86/avx-intrinsics-x86.ll b/test/CodeGen/X86/avx-intrinsics-x86.ll index 28a0272ecf02..206be2396cba 100644 --- a/test/CodeGen/X86/avx-intrinsics-x86.ll +++ b/test/CodeGen/X86/avx-intrinsics-x86.ll @@ -1,8 +1,9 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-apple-darwin -march=x86 -mattr=avx,aes,pclmul | FileCheck %s define <2 x i64> @test_x86_aesni_aesdec(<2 x i64> %a0, <2 x i64> %a1) { ; CHECK-LABEL: test_x86_aesni_aesdec: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vaesdec %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x i64> @llvm.x86.aesni.aesdec(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1] @@ -13,7 +14,7 @@ declare <2 x i64> @llvm.x86.aesni.aesdec(<2 x i64>, <2 x i64>) nounwind readnone define <2 x i64> @test_x86_aesni_aesdeclast(<2 x i64> %a0, <2 x i64> %a1) { ; CHECK-LABEL: test_x86_aesni_aesdeclast: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vaesdeclast %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x i64> @llvm.x86.aesni.aesdeclast(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1] @@ -24,7 +25,7 @@ declare <2 x i64> @llvm.x86.aesni.aesdeclast(<2 x i64>, <2 x i64>) nounwind read define <2 x i64> @test_x86_aesni_aesenc(<2 x i64> %a0, <2 x i64> %a1) { ; CHECK-LABEL: test_x86_aesni_aesenc: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vaesenc %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x i64> @llvm.x86.aesni.aesenc(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1] @@ -35,7 +36,7 @@ declare <2 x i64> @llvm.x86.aesni.aesenc(<2 x i64>, <2 x i64>) nounwind readnone define <2 x i64> @test_x86_aesni_aesenclast(<2 x i64> %a0, <2 x i64> %a1) { ; CHECK-LABEL: test_x86_aesni_aesenclast: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vaesenclast %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x i64> @llvm.x86.aesni.aesenclast(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1] @@ -46,7 +47,7 @@ declare <2 x i64> @llvm.x86.aesni.aesenclast(<2 x i64>, <2 x i64>) nounwind read define <2 x i64> @test_x86_aesni_aesimc(<2 x i64> %a0) { ; CHECK-LABEL: test_x86_aesni_aesimc: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vaesimc %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x i64> @llvm.x86.aesni.aesimc(<2 x i64> %a0) ; <<2 x i64>> [#uses=1] @@ -57,7 +58,7 @@ declare <2 x i64> @llvm.x86.aesni.aesimc(<2 x i64>) nounwind readnone define <2 x i64> @test_x86_aesni_aeskeygenassist(<2 x i64> %a0) { ; CHECK-LABEL: test_x86_aesni_aeskeygenassist: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vaeskeygenassist $7, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x i64> @llvm.x86.aesni.aeskeygenassist(<2 x i64> %a0, i8 7) ; <<2 x i64>> [#uses=1] @@ -68,7 +69,7 @@ declare <2 x i64> @llvm.x86.aesni.aeskeygenassist(<2 x i64>, i8) nounwind readno define <2 x double> @test_x86_sse2_add_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_add_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vaddsd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse2.add.sd(<2 x double> %a0, <2 x double> %a1) ; <<2 x double>> [#uses=1] @@ -79,7 +80,7 @@ declare <2 x double> @llvm.x86.sse2.add.sd(<2 x double>, <2 x double>) nounwind define <2 x double> @test_x86_sse2_cmp_pd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_cmp_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcmpordpd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse2.cmp.pd(<2 x double> %a0, <2 x double> %a1, i8 7) ; <<2 x double>> [#uses=1] @@ -90,7 +91,7 @@ declare <2 x double> @llvm.x86.sse2.cmp.pd(<2 x double>, <2 x double>, i8) nounw define <2 x double> @test_x86_sse2_cmp_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_cmp_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcmpordsd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %a0, <2 x double> %a1, i8 7) ; <<2 x double>> [#uses=1] @@ -101,7 +102,7 @@ declare <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double>, <2 x double>, i8) nounw define i32 @test_x86_sse2_comieq_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_comieq_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcomisd %xmm1, %xmm0 ; CHECK-NEXT: sete %al ; CHECK-NEXT: movzbl %al, %eax @@ -114,7 +115,7 @@ declare i32 @llvm.x86.sse2.comieq.sd(<2 x double>, <2 x double>) nounwind readno define i32 @test_x86_sse2_comige_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_comige_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcomisd %xmm1, %xmm0 ; CHECK-NEXT: setae %al ; CHECK-NEXT: movzbl %al, %eax @@ -127,7 +128,7 @@ declare i32 @llvm.x86.sse2.comige.sd(<2 x double>, <2 x double>) nounwind readno define i32 @test_x86_sse2_comigt_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_comigt_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcomisd %xmm1, %xmm0 ; CHECK-NEXT: seta %al ; CHECK-NEXT: movzbl %al, %eax @@ -140,7 +141,7 @@ declare i32 @llvm.x86.sse2.comigt.sd(<2 x double>, <2 x double>) nounwind readno define i32 @test_x86_sse2_comile_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_comile_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcomisd %xmm1, %xmm0 ; CHECK-NEXT: setbe %al ; CHECK-NEXT: movzbl %al, %eax @@ -153,7 +154,7 @@ declare i32 @llvm.x86.sse2.comile.sd(<2 x double>, <2 x double>) nounwind readno define i32 @test_x86_sse2_comilt_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_comilt_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcomisd %xmm1, %xmm0 ; CHECK-NEXT: sbbl %eax, %eax ; CHECK-NEXT: andl $1, %eax @@ -166,7 +167,7 @@ declare i32 @llvm.x86.sse2.comilt.sd(<2 x double>, <2 x double>) nounwind readno define i32 @test_x86_sse2_comineq_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_comineq_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcomisd %xmm1, %xmm0 ; CHECK-NEXT: setne %al ; CHECK-NEXT: movzbl %al, %eax @@ -179,7 +180,7 @@ declare i32 @llvm.x86.sse2.comineq.sd(<2 x double>, <2 x double>) nounwind readn define <2 x double> @test_x86_sse2_cvtdq2pd(<4 x i32> %a0) { ; CHECK-LABEL: test_x86_sse2_cvtdq2pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvtdq2pd %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse2.cvtdq2pd(<4 x i32> %a0) ; <<2 x double>> [#uses=1] @@ -190,7 +191,7 @@ declare <2 x double> @llvm.x86.sse2.cvtdq2pd(<4 x i32>) nounwind readnone define <4 x float> @test_x86_sse2_cvtdq2ps(<4 x i32> %a0) { ; CHECK-LABEL: test_x86_sse2_cvtdq2ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvtdq2ps %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse2.cvtdq2ps(<4 x i32> %a0) ; <<4 x float>> [#uses=1] @@ -201,7 +202,7 @@ declare <4 x float> @llvm.x86.sse2.cvtdq2ps(<4 x i32>) nounwind readnone define <4 x i32> @test_x86_sse2_cvtpd2dq(<2 x double> %a0) { ; CHECK-LABEL: test_x86_sse2_cvtpd2dq: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvtpd2dq %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double> %a0) ; <<4 x i32>> [#uses=1] @@ -212,7 +213,7 @@ declare <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double>) nounwind readnone define <4 x float> @test_x86_sse2_cvtpd2ps(<2 x double> %a0) { ; CHECK-LABEL: test_x86_sse2_cvtpd2ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvtpd2ps %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse2.cvtpd2ps(<2 x double> %a0) ; <<4 x float>> [#uses=1] @@ -223,7 +224,7 @@ declare <4 x float> @llvm.x86.sse2.cvtpd2ps(<2 x double>) nounwind readnone define <4 x i32> @test_x86_sse2_cvtps2dq(<4 x float> %a0) { ; CHECK-LABEL: test_x86_sse2_cvtps2dq: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvtps2dq %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.sse2.cvtps2dq(<4 x float> %a0) ; <<4 x i32>> [#uses=1] @@ -234,7 +235,7 @@ declare <4 x i32> @llvm.x86.sse2.cvtps2dq(<4 x float>) nounwind readnone define <2 x double> @test_x86_sse2_cvtps2pd(<4 x float> %a0) { ; CHECK-LABEL: test_x86_sse2_cvtps2pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvtps2pd %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse2.cvtps2pd(<4 x float> %a0) ; <<2 x double>> [#uses=1] @@ -245,7 +246,7 @@ declare <2 x double> @llvm.x86.sse2.cvtps2pd(<4 x float>) nounwind readnone define i32 @test_x86_sse2_cvtsd2si(<2 x double> %a0) { ; CHECK-LABEL: test_x86_sse2_cvtsd2si: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvtsd2si %xmm0, %eax ; CHECK-NEXT: retl %res = call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> %a0) ; [#uses=1] @@ -256,7 +257,7 @@ declare i32 @llvm.x86.sse2.cvtsd2si(<2 x double>) nounwind readnone define <4 x float> @test_x86_sse2_cvtsd2ss(<4 x float> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_cvtsd2ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvtsd2ss %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse2.cvtsd2ss(<4 x float> %a0, <2 x double> %a1) ; <<4 x float>> [#uses=1] @@ -267,7 +268,7 @@ declare <4 x float> @llvm.x86.sse2.cvtsd2ss(<4 x float>, <2 x double>) nounwind define <2 x double> @test_x86_sse2_cvtsi2sd(<2 x double> %a0) { ; CHECK-LABEL: test_x86_sse2_cvtsi2sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl $7, %eax ; CHECK-NEXT: vcvtsi2sdl %eax, %xmm0, %xmm0 ; CHECK-NEXT: retl @@ -279,7 +280,7 @@ declare <2 x double> @llvm.x86.sse2.cvtsi2sd(<2 x double>, i32) nounwind readnon define <2 x double> @test_x86_sse2_cvtss2sd(<2 x double> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse2_cvtss2sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvtss2sd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse2.cvtss2sd(<2 x double> %a0, <4 x float> %a1) ; <<2 x double>> [#uses=1] @@ -290,7 +291,7 @@ declare <2 x double> @llvm.x86.sse2.cvtss2sd(<2 x double>, <4 x float>) nounwind define <4 x i32> @test_x86_sse2_cvttpd2dq(<2 x double> %a0) { ; CHECK-LABEL: test_x86_sse2_cvttpd2dq: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvttpd2dq %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.sse2.cvttpd2dq(<2 x double> %a0) ; <<4 x i32>> [#uses=1] @@ -301,7 +302,7 @@ declare <4 x i32> @llvm.x86.sse2.cvttpd2dq(<2 x double>) nounwind readnone define <4 x i32> @test_x86_sse2_cvttps2dq(<4 x float> %a0) { ; CHECK-LABEL: test_x86_sse2_cvttps2dq: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvttps2dq %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.sse2.cvttps2dq(<4 x float> %a0) ; <<4 x i32>> [#uses=1] @@ -312,7 +313,7 @@ declare <4 x i32> @llvm.x86.sse2.cvttps2dq(<4 x float>) nounwind readnone define i32 @test_x86_sse2_cvttsd2si(<2 x double> %a0) { ; CHECK-LABEL: test_x86_sse2_cvttsd2si: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvttsd2si %xmm0, %eax ; CHECK-NEXT: retl %res = call i32 @llvm.x86.sse2.cvttsd2si(<2 x double> %a0) ; [#uses=1] @@ -323,7 +324,7 @@ declare i32 @llvm.x86.sse2.cvttsd2si(<2 x double>) nounwind readnone define <2 x double> @test_x86_sse2_div_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_div_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vdivsd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse2.div.sd(<2 x double> %a0, <2 x double> %a1) ; <<2 x double>> [#uses=1] @@ -335,7 +336,7 @@ declare <2 x double> @llvm.x86.sse2.div.sd(<2 x double>, <2 x double>) nounwind define <2 x double> @test_x86_sse2_max_pd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_max_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vmaxpd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse2.max.pd(<2 x double> %a0, <2 x double> %a1) ; <<2 x double>> [#uses=1] @@ -346,7 +347,7 @@ declare <2 x double> @llvm.x86.sse2.max.pd(<2 x double>, <2 x double>) nounwind define <2 x double> @test_x86_sse2_max_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_max_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vmaxsd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> %a0, <2 x double> %a1) ; <<2 x double>> [#uses=1] @@ -357,7 +358,7 @@ declare <2 x double> @llvm.x86.sse2.max.sd(<2 x double>, <2 x double>) nounwind define <2 x double> @test_x86_sse2_min_pd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_min_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vminpd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse2.min.pd(<2 x double> %a0, <2 x double> %a1) ; <<2 x double>> [#uses=1] @@ -368,7 +369,7 @@ declare <2 x double> @llvm.x86.sse2.min.pd(<2 x double>, <2 x double>) nounwind define <2 x double> @test_x86_sse2_min_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_min_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vminsd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> %a0, <2 x double> %a1) ; <<2 x double>> [#uses=1] @@ -379,7 +380,7 @@ declare <2 x double> @llvm.x86.sse2.min.sd(<2 x double>, <2 x double>) nounwind define i32 @test_x86_sse2_movmsk_pd(<2 x double> %a0) { ; CHECK-LABEL: test_x86_sse2_movmsk_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vmovmskpd %xmm0, %eax ; CHECK-NEXT: retl %res = call i32 @llvm.x86.sse2.movmsk.pd(<2 x double> %a0) ; [#uses=1] @@ -392,7 +393,7 @@ declare i32 @llvm.x86.sse2.movmsk.pd(<2 x double>) nounwind readnone define <2 x double> @test_x86_sse2_mul_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_mul_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vmulsd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse2.mul.sd(<2 x double> %a0, <2 x double> %a1) ; <<2 x double>> [#uses=1] @@ -403,7 +404,7 @@ declare <2 x double> @llvm.x86.sse2.mul.sd(<2 x double>, <2 x double>) nounwind define <8 x i16> @test_x86_sse2_packssdw_128(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: test_x86_sse2_packssdw_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpackssdw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> %a0, <4 x i32> %a1) ; <<8 x i16>> [#uses=1] @@ -414,7 +415,7 @@ declare <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32>, <4 x i32>) nounwind rea define <16 x i8> @test_x86_sse2_packsswb_128(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_sse2_packsswb_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpacksswb %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> %a0, <8 x i16> %a1) ; <<16 x i8>> [#uses=1] @@ -425,7 +426,7 @@ declare <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16>, <8 x i16>) nounwind rea define <16 x i8> @test_x86_sse2_packuswb_128(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_sse2_packuswb_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpackuswb %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> %a0, <8 x i16> %a1) ; <<16 x i8>> [#uses=1] @@ -436,7 +437,7 @@ declare <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16>, <8 x i16>) nounwind rea define <16 x i8> @test_x86_sse2_padds_b(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse2_padds_b: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpaddsb %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %a0, <16 x i8> %a1) ; <<16 x i8>> [#uses=1] @@ -447,7 +448,7 @@ declare <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8>, <16 x i8>) nounwind readnone define <8 x i16> @test_x86_sse2_padds_w(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_sse2_padds_w: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpaddsw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -458,7 +459,7 @@ declare <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16>, <8 x i16>) nounwind readnone define <16 x i8> @test_x86_sse2_paddus_b(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse2_paddus_b: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpaddusb %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <16 x i8> @llvm.x86.sse2.paddus.b(<16 x i8> %a0, <16 x i8> %a1) ; <<16 x i8>> [#uses=1] @@ -469,7 +470,7 @@ declare <16 x i8> @llvm.x86.sse2.paddus.b(<16 x i8>, <16 x i8>) nounwind readnon define <8 x i16> @test_x86_sse2_paddus_w(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_sse2_paddus_w: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpaddusw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse2.paddus.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -480,7 +481,7 @@ declare <8 x i16> @llvm.x86.sse2.paddus.w(<8 x i16>, <8 x i16>) nounwind readnon define <16 x i8> @test_x86_sse2_pavg_b(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse2_pavg_b: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpavgb %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <16 x i8> @llvm.x86.sse2.pavg.b(<16 x i8> %a0, <16 x i8> %a1) ; <<16 x i8>> [#uses=1] @@ -491,7 +492,7 @@ declare <16 x i8> @llvm.x86.sse2.pavg.b(<16 x i8>, <16 x i8>) nounwind readnone define <8 x i16> @test_x86_sse2_pavg_w(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_sse2_pavg_w: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpavgw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse2.pavg.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -502,7 +503,7 @@ declare <8 x i16> @llvm.x86.sse2.pavg.w(<8 x i16>, <8 x i16>) nounwind readnone define <4 x i32> @test_x86_sse2_pmadd_wd(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_sse2_pmadd_wd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a0, <8 x i16> %a1) ; <<4 x i32>> [#uses=1] @@ -513,7 +514,7 @@ declare <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>) nounwind readnon define <8 x i16> @test_x86_sse2_pmaxs_w(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_sse2_pmaxs_w: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse2.pmaxs.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -524,7 +525,7 @@ declare <8 x i16> @llvm.x86.sse2.pmaxs.w(<8 x i16>, <8 x i16>) nounwind readnone define <16 x i8> @test_x86_sse2_pmaxu_b(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse2_pmaxu_b: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmaxub %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <16 x i8> @llvm.x86.sse2.pmaxu.b(<16 x i8> %a0, <16 x i8> %a1) ; <<16 x i8>> [#uses=1] @@ -535,7 +536,7 @@ declare <16 x i8> @llvm.x86.sse2.pmaxu.b(<16 x i8>, <16 x i8>) nounwind readnone define <8 x i16> @test_x86_sse2_pmins_w(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_sse2_pmins_w: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpminsw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse2.pmins.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -546,7 +547,7 @@ declare <8 x i16> @llvm.x86.sse2.pmins.w(<8 x i16>, <8 x i16>) nounwind readnone define <16 x i8> @test_x86_sse2_pminu_b(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse2_pminu_b: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpminub %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <16 x i8> @llvm.x86.sse2.pminu.b(<16 x i8> %a0, <16 x i8> %a1) ; <<16 x i8>> [#uses=1] @@ -557,7 +558,7 @@ declare <16 x i8> @llvm.x86.sse2.pminu.b(<16 x i8>, <16 x i8>) nounwind readnone define i32 @test_x86_sse2_pmovmskb_128(<16 x i8> %a0) { ; CHECK-LABEL: test_x86_sse2_pmovmskb_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmovmskb %xmm0, %eax ; CHECK-NEXT: retl %res = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %a0) ; [#uses=1] @@ -568,7 +569,7 @@ declare i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8>) nounwind readnone define <8 x i16> @test_x86_sse2_pmulh_w(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_sse2_pmulh_w: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmulhw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -579,7 +580,7 @@ declare <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16>, <8 x i16>) nounwind readnone define <8 x i16> @test_x86_sse2_pmulhu_w(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_sse2_pmulhu_w: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmulhuw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -590,7 +591,7 @@ declare <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16>, <8 x i16>) nounwind readnon define <2 x i64> @test_x86_sse2_pmulu_dq(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: test_x86_sse2_pmulu_dq: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmuludq %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x i64> @llvm.x86.sse2.pmulu.dq(<4 x i32> %a0, <4 x i32> %a1) ; <<2 x i64>> [#uses=1] @@ -601,7 +602,7 @@ declare <2 x i64> @llvm.x86.sse2.pmulu.dq(<4 x i32>, <4 x i32>) nounwind readnon define <2 x i64> @test_x86_sse2_psad_bw(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse2_psad_bw: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsadbw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> %a0, <16 x i8> %a1) ; <<2 x i64>> [#uses=1] @@ -612,7 +613,7 @@ declare <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8>, <16 x i8>) nounwind readnone define <4 x i32> @test_x86_sse2_psll_d(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: test_x86_sse2_psll_d: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpslld %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1] @@ -623,7 +624,7 @@ declare <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32>, <4 x i32>) nounwind readnone define <2 x i64> @test_x86_sse2_psll_q(<2 x i64> %a0, <2 x i64> %a1) { ; CHECK-LABEL: test_x86_sse2_psll_q: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsllq %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1] @@ -634,7 +635,7 @@ declare <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64>, <2 x i64>) nounwind readnone define <8 x i16> @test_x86_sse2_psll_w(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_sse2_psll_w: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsllw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -645,7 +646,7 @@ declare <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16>, <8 x i16>) nounwind readnone define <4 x i32> @test_x86_sse2_pslli_d(<4 x i32> %a0) { ; CHECK-LABEL: test_x86_sse2_pslli_d: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpslld $7, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %a0, i32 7) ; <<4 x i32>> [#uses=1] @@ -656,7 +657,7 @@ declare <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32>, i32) nounwind readnone define <2 x i64> @test_x86_sse2_pslli_q(<2 x i64> %a0) { ; CHECK-LABEL: test_x86_sse2_pslli_q: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsllq $7, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %a0, i32 7) ; <<2 x i64>> [#uses=1] @@ -667,7 +668,7 @@ declare <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64>, i32) nounwind readnone define <8 x i16> @test_x86_sse2_pslli_w(<8 x i16> %a0) { ; CHECK-LABEL: test_x86_sse2_pslli_w: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsllw $7, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16> %a0, i32 7) ; <<8 x i16>> [#uses=1] @@ -678,7 +679,7 @@ declare <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16>, i32) nounwind readnone define <4 x i32> @test_x86_sse2_psra_d(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: test_x86_sse2_psra_d: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsrad %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1] @@ -689,7 +690,7 @@ declare <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32>, <4 x i32>) nounwind readnone define <8 x i16> @test_x86_sse2_psra_w(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_sse2_psra_w: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsraw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -700,7 +701,7 @@ declare <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16>, <8 x i16>) nounwind readnone define <4 x i32> @test_x86_sse2_psrai_d(<4 x i32> %a0) { ; CHECK-LABEL: test_x86_sse2_psrai_d: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsrad $7, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %a0, i32 7) ; <<4 x i32>> [#uses=1] @@ -711,7 +712,7 @@ declare <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32>, i32) nounwind readnone define <8 x i16> @test_x86_sse2_psrai_w(<8 x i16> %a0) { ; CHECK-LABEL: test_x86_sse2_psrai_w: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsraw $7, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %a0, i32 7) ; <<8 x i16>> [#uses=1] @@ -722,7 +723,7 @@ declare <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16>, i32) nounwind readnone define <4 x i32> @test_x86_sse2_psrl_d(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: test_x86_sse2_psrl_d: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsrld %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1] @@ -733,7 +734,7 @@ declare <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32>, <4 x i32>) nounwind readnone define <2 x i64> @test_x86_sse2_psrl_q(<2 x i64> %a0, <2 x i64> %a1) { ; CHECK-LABEL: test_x86_sse2_psrl_q: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsrlq %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1] @@ -744,7 +745,7 @@ declare <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64>, <2 x i64>) nounwind readnone define <8 x i16> @test_x86_sse2_psrl_w(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_sse2_psrl_w: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsrlw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -755,7 +756,7 @@ declare <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16>, <8 x i16>) nounwind readnone define <4 x i32> @test_x86_sse2_psrli_d(<4 x i32> %a0) { ; CHECK-LABEL: test_x86_sse2_psrli_d: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsrld $7, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> %a0, i32 7) ; <<4 x i32>> [#uses=1] @@ -766,7 +767,7 @@ declare <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32>, i32) nounwind readnone define <2 x i64> @test_x86_sse2_psrli_q(<2 x i64> %a0) { ; CHECK-LABEL: test_x86_sse2_psrli_q: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsrlq $7, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %a0, i32 7) ; <<2 x i64>> [#uses=1] @@ -777,7 +778,7 @@ declare <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64>, i32) nounwind readnone define <8 x i16> @test_x86_sse2_psrli_w(<8 x i16> %a0) { ; CHECK-LABEL: test_x86_sse2_psrli_w: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsrlw $7, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16> %a0, i32 7) ; <<8 x i16>> [#uses=1] @@ -788,7 +789,7 @@ declare <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16>, i32) nounwind readnone define <16 x i8> @test_x86_sse2_psubs_b(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse2_psubs_b: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsubsb %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> %a0, <16 x i8> %a1) ; <<16 x i8>> [#uses=1] @@ -799,7 +800,7 @@ declare <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8>, <16 x i8>) nounwind readnone define <8 x i16> @test_x86_sse2_psubs_w(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_sse2_psubs_w: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsubsw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -810,7 +811,7 @@ declare <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16>, <8 x i16>) nounwind readnone define <16 x i8> @test_x86_sse2_psubus_b(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse2_psubus_b: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsubusb %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <16 x i8> @llvm.x86.sse2.psubus.b(<16 x i8> %a0, <16 x i8> %a1) ; <<16 x i8>> [#uses=1] @@ -821,7 +822,7 @@ declare <16 x i8> @llvm.x86.sse2.psubus.b(<16 x i8>, <16 x i8>) nounwind readnon define <8 x i16> @test_x86_sse2_psubus_w(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_sse2_psubus_w: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsubusw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse2.psubus.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -832,7 +833,7 @@ declare <8 x i16> @llvm.x86.sse2.psubus.w(<8 x i16>, <8 x i16>) nounwind readnon define <2 x double> @test_x86_sse2_sqrt_pd(<2 x double> %a0) { ; CHECK-LABEL: test_x86_sse2_sqrt_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vsqrtpd %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse2.sqrt.pd(<2 x double> %a0) ; <<2 x double>> [#uses=1] @@ -843,7 +844,7 @@ declare <2 x double> @llvm.x86.sse2.sqrt.pd(<2 x double>) nounwind readnone define <2 x double> @test_x86_sse2_sqrt_sd(<2 x double> %a0) { ; CHECK-LABEL: test_x86_sse2_sqrt_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vsqrtsd %xmm0, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double> %a0) ; <<2 x double>> [#uses=1] @@ -854,7 +855,7 @@ declare <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double>) nounwind readnone define void @test_x86_sse2_storel_dq(i8* %a0, <4 x i32> %a1) { ; CHECK-LABEL: test_x86_sse2_storel_dq: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vmovlps %xmm0, (%eax) ; CHECK-NEXT: retl @@ -867,7 +868,7 @@ declare void @llvm.x86.sse2.storel.dq(i8*, <4 x i32>) nounwind define void @test_x86_sse2_storeu_dq(i8* %a0, <16 x i8> %a1) { ; add operation forces the execution domain. ; CHECK-LABEL: test_x86_sse2_storeu_dq: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vpaddb LCPI77_0, %xmm0, %xmm0 ; CHECK-NEXT: vmovdqu %xmm0, (%eax) @@ -882,7 +883,7 @@ declare void @llvm.x86.sse2.storeu.dq(i8*, <16 x i8>) nounwind define void @test_x86_sse2_storeu_pd(i8* %a0, <2 x double> %a1) { ; fadd operation forces the execution domain. ; CHECK-LABEL: test_x86_sse2_storeu_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero ; CHECK-NEXT: vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6,7] @@ -898,7 +899,7 @@ declare void @llvm.x86.sse2.storeu.pd(i8*, <2 x double>) nounwind define <2 x double> @test_x86_sse2_sub_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_sub_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vsubsd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse2.sub.sd(<2 x double> %a0, <2 x double> %a1) ; <<2 x double>> [#uses=1] @@ -909,7 +910,7 @@ declare <2 x double> @llvm.x86.sse2.sub.sd(<2 x double>, <2 x double>) nounwind define i32 @test_x86_sse2_ucomieq_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_ucomieq_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vucomisd %xmm1, %xmm0 ; CHECK-NEXT: sete %al ; CHECK-NEXT: movzbl %al, %eax @@ -922,7 +923,7 @@ declare i32 @llvm.x86.sse2.ucomieq.sd(<2 x double>, <2 x double>) nounwind readn define i32 @test_x86_sse2_ucomige_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_ucomige_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vucomisd %xmm1, %xmm0 ; CHECK-NEXT: setae %al ; CHECK-NEXT: movzbl %al, %eax @@ -935,7 +936,7 @@ declare i32 @llvm.x86.sse2.ucomige.sd(<2 x double>, <2 x double>) nounwind readn define i32 @test_x86_sse2_ucomigt_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_ucomigt_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vucomisd %xmm1, %xmm0 ; CHECK-NEXT: seta %al ; CHECK-NEXT: movzbl %al, %eax @@ -948,7 +949,7 @@ declare i32 @llvm.x86.sse2.ucomigt.sd(<2 x double>, <2 x double>) nounwind readn define i32 @test_x86_sse2_ucomile_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_ucomile_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vucomisd %xmm1, %xmm0 ; CHECK-NEXT: setbe %al ; CHECK-NEXT: movzbl %al, %eax @@ -961,7 +962,7 @@ declare i32 @llvm.x86.sse2.ucomile.sd(<2 x double>, <2 x double>) nounwind readn define i32 @test_x86_sse2_ucomilt_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_ucomilt_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vucomisd %xmm1, %xmm0 ; CHECK-NEXT: sbbl %eax, %eax ; CHECK-NEXT: andl $1, %eax @@ -974,7 +975,7 @@ declare i32 @llvm.x86.sse2.ucomilt.sd(<2 x double>, <2 x double>) nounwind readn define i32 @test_x86_sse2_ucomineq_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse2_ucomineq_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vucomisd %xmm1, %xmm0 ; CHECK-NEXT: setne %al ; CHECK-NEXT: movzbl %al, %eax @@ -987,7 +988,7 @@ declare i32 @llvm.x86.sse2.ucomineq.sd(<2 x double>, <2 x double>) nounwind read define <2 x double> @test_x86_sse3_addsub_pd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse3_addsub_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vaddsubpd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> %a0, <2 x double> %a1) ; <<2 x double>> [#uses=1] @@ -998,7 +999,7 @@ declare <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double>, <2 x double>) nounwi define <4 x float> @test_x86_sse3_addsub_ps(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse3_addsub_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vaddsubps %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1] @@ -1009,7 +1010,7 @@ declare <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float>, <4 x float>) nounwind define <2 x double> @test_x86_sse3_hadd_pd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse3_hadd_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vhaddpd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse3.hadd.pd(<2 x double> %a0, <2 x double> %a1) ; <<2 x double>> [#uses=1] @@ -1020,7 +1021,7 @@ declare <2 x double> @llvm.x86.sse3.hadd.pd(<2 x double>, <2 x double>) nounwind define <4 x float> @test_x86_sse3_hadd_ps(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse3_hadd_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vhaddps %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse3.hadd.ps(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1] @@ -1031,7 +1032,7 @@ declare <4 x float> @llvm.x86.sse3.hadd.ps(<4 x float>, <4 x float>) nounwind re define <2 x double> @test_x86_sse3_hsub_pd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse3_hsub_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vhsubpd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse3.hsub.pd(<2 x double> %a0, <2 x double> %a1) ; <<2 x double>> [#uses=1] @@ -1042,7 +1043,7 @@ declare <2 x double> @llvm.x86.sse3.hsub.pd(<2 x double>, <2 x double>) nounwind define <4 x float> @test_x86_sse3_hsub_ps(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse3_hsub_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vhsubps %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse3.hsub.ps(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1] @@ -1053,7 +1054,7 @@ declare <4 x float> @llvm.x86.sse3.hsub.ps(<4 x float>, <4 x float>) nounwind re define <16 x i8> @test_x86_sse3_ldu_dq(i8* %a0) { ; CHECK-LABEL: test_x86_sse3_ldu_dq: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vlddqu (%eax), %xmm0 ; CHECK-NEXT: retl @@ -1065,7 +1066,7 @@ declare <16 x i8> @llvm.x86.sse3.ldu.dq(i8*) nounwind readonly define <2 x double> @test_x86_sse41_blendvpd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) { ; CHECK-LABEL: test_x86_sse41_blendvpd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) ; <<2 x double>> [#uses=1] @@ -1076,7 +1077,7 @@ declare <2 x double> @llvm.x86.sse41.blendvpd(<2 x double>, <2 x double>, <2 x d define <4 x float> @test_x86_sse41_blendvps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) { ; CHECK-LABEL: test_x86_sse41_blendvps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) ; <<4 x float>> [#uses=1] @@ -1087,7 +1088,7 @@ declare <4 x float> @llvm.x86.sse41.blendvps(<4 x float>, <4 x float>, <4 x floa define <2 x double> @test_x86_sse41_dppd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse41_dppd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vdppd $7, %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse41.dppd(<2 x double> %a0, <2 x double> %a1, i8 7) ; <<2 x double>> [#uses=1] @@ -1098,7 +1099,7 @@ declare <2 x double> @llvm.x86.sse41.dppd(<2 x double>, <2 x double>, i8) nounwi define <4 x float> @test_x86_sse41_dpps(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse41_dpps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vdpps $7, %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse41.dpps(<4 x float> %a0, <4 x float> %a1, i8 7) ; <<4 x float>> [#uses=1] @@ -1109,7 +1110,7 @@ declare <4 x float> @llvm.x86.sse41.dpps(<4 x float>, <4 x float>, i8) nounwind define <4 x float> @test_x86_sse41_insertps(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse41_insertps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vinsertps {{.*#+}} xmm0 = zero,zero,zero,xmm0[3] ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %a0, <4 x float> %a1, i8 7) ; <<4 x float>> [#uses=1] @@ -1121,7 +1122,7 @@ declare <4 x float> @llvm.x86.sse41.insertps(<4 x float>, <4 x float>, i8) nounw define <8 x i16> @test_x86_sse41_mpsadbw(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse41_mpsadbw: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vmpsadbw $7, %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8> %a0, <16 x i8> %a1, i8 7) ; <<8 x i16>> [#uses=1] @@ -1132,7 +1133,7 @@ declare <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8>, <16 x i8>, i8) nounwind rea define <8 x i16> @test_x86_sse41_packusdw(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: test_x86_sse41_packusdw: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpackusdw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> %a0, <4 x i32> %a1) ; <<8 x i16>> [#uses=1] @@ -1143,7 +1144,7 @@ declare <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32>, <4 x i32>) nounwind readno define <16 x i8> @test_x86_sse41_pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2) { ; CHECK-LABEL: test_x86_sse41_pblendvb: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2) ; <<16 x i8>> [#uses=1] @@ -1154,7 +1155,7 @@ declare <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8>, <16 x i8>, <16 x i8>) noun define <8 x i16> @test_x86_sse41_phminposuw(<8 x i16> %a0) { ; CHECK-LABEL: test_x86_sse41_phminposuw: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vphminposuw %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse41.phminposuw(<8 x i16> %a0) ; <<8 x i16>> [#uses=1] @@ -1165,7 +1166,7 @@ declare <8 x i16> @llvm.x86.sse41.phminposuw(<8 x i16>) nounwind readnone define <16 x i8> @test_x86_sse41_pmaxsb(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse41_pmaxsb: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <16 x i8> @llvm.x86.sse41.pmaxsb(<16 x i8> %a0, <16 x i8> %a1) ; <<16 x i8>> [#uses=1] @@ -1176,7 +1177,7 @@ declare <16 x i8> @llvm.x86.sse41.pmaxsb(<16 x i8>, <16 x i8>) nounwind readnone define <4 x i32> @test_x86_sse41_pmaxsd(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: test_x86_sse41_pmaxsd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.sse41.pmaxsd(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1] @@ -1187,7 +1188,7 @@ declare <4 x i32> @llvm.x86.sse41.pmaxsd(<4 x i32>, <4 x i32>) nounwind readnone define <4 x i32> @test_x86_sse41_pmaxud(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: test_x86_sse41_pmaxud: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmaxud %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.sse41.pmaxud(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1] @@ -1198,7 +1199,7 @@ declare <4 x i32> @llvm.x86.sse41.pmaxud(<4 x i32>, <4 x i32>) nounwind readnone define <8 x i16> @test_x86_sse41_pmaxuw(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_sse41_pmaxuw: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmaxuw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse41.pmaxuw(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -1209,7 +1210,7 @@ declare <8 x i16> @llvm.x86.sse41.pmaxuw(<8 x i16>, <8 x i16>) nounwind readnone define <16 x i8> @test_x86_sse41_pminsb(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse41_pminsb: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpminsb %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <16 x i8> @llvm.x86.sse41.pminsb(<16 x i8> %a0, <16 x i8> %a1) ; <<16 x i8>> [#uses=1] @@ -1220,7 +1221,7 @@ declare <16 x i8> @llvm.x86.sse41.pminsb(<16 x i8>, <16 x i8>) nounwind readnone define <4 x i32> @test_x86_sse41_pminsd(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: test_x86_sse41_pminsd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpminsd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.sse41.pminsd(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1] @@ -1231,7 +1232,7 @@ declare <4 x i32> @llvm.x86.sse41.pminsd(<4 x i32>, <4 x i32>) nounwind readnone define <4 x i32> @test_x86_sse41_pminud(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: test_x86_sse41_pminud: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpminud %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.sse41.pminud(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1] @@ -1242,7 +1243,7 @@ declare <4 x i32> @llvm.x86.sse41.pminud(<4 x i32>, <4 x i32>) nounwind readnone define <8 x i16> @test_x86_sse41_pminuw(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_sse41_pminuw: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpminuw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse41.pminuw(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -1251,75 +1252,9 @@ define <8 x i16> @test_x86_sse41_pminuw(<8 x i16> %a0, <8 x i16> %a1) { declare <8 x i16> @llvm.x86.sse41.pminuw(<8 x i16>, <8 x i16>) nounwind readnone -define <4 x i32> @test_x86_sse41_pmovsxbd(<16 x i8> %a0) { -; CHECK-LABEL: test_x86_sse41_pmovsxbd: -; CHECK: # BB#0: -; CHECK-NEXT: vpmovsxbd %xmm0, %xmm0 -; CHECK-NEXT: retl - %res = call <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8> %a0) ; <<4 x i32>> [#uses=1] - ret <4 x i32> %res -} -declare <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8>) nounwind readnone - - -define <2 x i64> @test_x86_sse41_pmovsxbq(<16 x i8> %a0) { -; CHECK-LABEL: test_x86_sse41_pmovsxbq: -; CHECK: # BB#0: -; CHECK-NEXT: vpmovsxbq %xmm0, %xmm0 -; CHECK-NEXT: retl - %res = call <2 x i64> @llvm.x86.sse41.pmovsxbq(<16 x i8> %a0) ; <<2 x i64>> [#uses=1] - ret <2 x i64> %res -} -declare <2 x i64> @llvm.x86.sse41.pmovsxbq(<16 x i8>) nounwind readnone - - -define <8 x i16> @test_x86_sse41_pmovsxbw(<16 x i8> %a0) { -; CHECK-LABEL: test_x86_sse41_pmovsxbw: -; CHECK: # BB#0: -; CHECK-NEXT: vpmovsxbw %xmm0, %xmm0 -; CHECK-NEXT: retl - %res = call <8 x i16> @llvm.x86.sse41.pmovsxbw(<16 x i8> %a0) ; <<8 x i16>> [#uses=1] - ret <8 x i16> %res -} -declare <8 x i16> @llvm.x86.sse41.pmovsxbw(<16 x i8>) nounwind readnone - - -define <2 x i64> @test_x86_sse41_pmovsxdq(<4 x i32> %a0) { -; CHECK-LABEL: test_x86_sse41_pmovsxdq: -; CHECK: # BB#0: -; CHECK-NEXT: vpmovsxdq %xmm0, %xmm0 -; CHECK-NEXT: retl - %res = call <2 x i64> @llvm.x86.sse41.pmovsxdq(<4 x i32> %a0) ; <<2 x i64>> [#uses=1] - ret <2 x i64> %res -} -declare <2 x i64> @llvm.x86.sse41.pmovsxdq(<4 x i32>) nounwind readnone - - -define <4 x i32> @test_x86_sse41_pmovsxwd(<8 x i16> %a0) { -; CHECK-LABEL: test_x86_sse41_pmovsxwd: -; CHECK: # BB#0: -; CHECK-NEXT: vpmovsxwd %xmm0, %xmm0 -; CHECK-NEXT: retl - %res = call <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16> %a0) ; <<4 x i32>> [#uses=1] - ret <4 x i32> %res -} -declare <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16>) nounwind readnone - - -define <2 x i64> @test_x86_sse41_pmovsxwq(<8 x i16> %a0) { -; CHECK-LABEL: test_x86_sse41_pmovsxwq: -; CHECK: # BB#0: -; CHECK-NEXT: vpmovsxwq %xmm0, %xmm0 -; CHECK-NEXT: retl - %res = call <2 x i64> @llvm.x86.sse41.pmovsxwq(<8 x i16> %a0) ; <<2 x i64>> [#uses=1] - ret <2 x i64> %res -} -declare <2 x i64> @llvm.x86.sse41.pmovsxwq(<8 x i16>) nounwind readnone - - define <4 x i32> @test_x86_sse41_pmovzxbd(<16 x i8> %a0) { ; CHECK-LABEL: test_x86_sse41_pmovzxbd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.sse41.pmovzxbd(<16 x i8> %a0) ; <<4 x i32>> [#uses=1] @@ -1330,7 +1265,7 @@ declare <4 x i32> @llvm.x86.sse41.pmovzxbd(<16 x i8>) nounwind readnone define <2 x i64> @test_x86_sse41_pmovzxbq(<16 x i8> %a0) { ; CHECK-LABEL: test_x86_sse41_pmovzxbq: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero ; CHECK-NEXT: retl %res = call <2 x i64> @llvm.x86.sse41.pmovzxbq(<16 x i8> %a0) ; <<2 x i64>> [#uses=1] @@ -1341,7 +1276,7 @@ declare <2 x i64> @llvm.x86.sse41.pmovzxbq(<16 x i8>) nounwind readnone define <8 x i16> @test_x86_sse41_pmovzxbw(<16 x i8> %a0) { ; CHECK-LABEL: test_x86_sse41_pmovzxbw: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.sse41.pmovzxbw(<16 x i8> %a0) ; <<8 x i16>> [#uses=1] @@ -1352,7 +1287,7 @@ declare <8 x i16> @llvm.x86.sse41.pmovzxbw(<16 x i8>) nounwind readnone define <2 x i64> @test_x86_sse41_pmovzxdq(<4 x i32> %a0) { ; CHECK-LABEL: test_x86_sse41_pmovzxdq: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero ; CHECK-NEXT: retl %res = call <2 x i64> @llvm.x86.sse41.pmovzxdq(<4 x i32> %a0) ; <<2 x i64>> [#uses=1] @@ -1363,7 +1298,7 @@ declare <2 x i64> @llvm.x86.sse41.pmovzxdq(<4 x i32>) nounwind readnone define <4 x i32> @test_x86_sse41_pmovzxwd(<8 x i16> %a0) { ; CHECK-LABEL: test_x86_sse41_pmovzxwd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.sse41.pmovzxwd(<8 x i16> %a0) ; <<4 x i32>> [#uses=1] @@ -1374,7 +1309,7 @@ declare <4 x i32> @llvm.x86.sse41.pmovzxwd(<8 x i16>) nounwind readnone define <2 x i64> @test_x86_sse41_pmovzxwq(<8 x i16> %a0) { ; CHECK-LABEL: test_x86_sse41_pmovzxwq: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero ; CHECK-NEXT: retl %res = call <2 x i64> @llvm.x86.sse41.pmovzxwq(<8 x i16> %a0) ; <<2 x i64>> [#uses=1] @@ -1385,7 +1320,7 @@ declare <2 x i64> @llvm.x86.sse41.pmovzxwq(<8 x i16>) nounwind readnone define <2 x i64> @test_x86_sse41_pmuldq(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: test_x86_sse41_pmuldq: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmuldq %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x i64> @llvm.x86.sse41.pmuldq(<4 x i32> %a0, <4 x i32> %a1) ; <<2 x i64>> [#uses=1] @@ -1396,7 +1331,7 @@ declare <2 x i64> @llvm.x86.sse41.pmuldq(<4 x i32>, <4 x i32>) nounwind readnone define i32 @test_x86_sse41_ptestc(<2 x i64> %a0, <2 x i64> %a1) { ; CHECK-LABEL: test_x86_sse41_ptestc: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vptest %xmm1, %xmm0 ; CHECK-NEXT: sbbl %eax, %eax ; CHECK-NEXT: andl $1, %eax @@ -1409,7 +1344,7 @@ declare i32 @llvm.x86.sse41.ptestc(<2 x i64>, <2 x i64>) nounwind readnone define i32 @test_x86_sse41_ptestnzc(<2 x i64> %a0, <2 x i64> %a1) { ; CHECK-LABEL: test_x86_sse41_ptestnzc: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vptest %xmm1, %xmm0 ; CHECK-NEXT: seta %al ; CHECK-NEXT: movzbl %al, %eax @@ -1422,7 +1357,7 @@ declare i32 @llvm.x86.sse41.ptestnzc(<2 x i64>, <2 x i64>) nounwind readnone define i32 @test_x86_sse41_ptestz(<2 x i64> %a0, <2 x i64> %a1) { ; CHECK-LABEL: test_x86_sse41_ptestz: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vptest %xmm1, %xmm0 ; CHECK-NEXT: sete %al ; CHECK-NEXT: movzbl %al, %eax @@ -1435,7 +1370,7 @@ declare i32 @llvm.x86.sse41.ptestz(<2 x i64>, <2 x i64>) nounwind readnone define <2 x double> @test_x86_sse41_round_pd(<2 x double> %a0) { ; CHECK-LABEL: test_x86_sse41_round_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vroundpd $7, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse41.round.pd(<2 x double> %a0, i32 7) ; <<2 x double>> [#uses=1] @@ -1446,7 +1381,7 @@ declare <2 x double> @llvm.x86.sse41.round.pd(<2 x double>, i32) nounwind readno define <4 x float> @test_x86_sse41_round_ps(<4 x float> %a0) { ; CHECK-LABEL: test_x86_sse41_round_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vroundps $7, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse41.round.ps(<4 x float> %a0, i32 7) ; <<4 x float>> [#uses=1] @@ -1457,7 +1392,7 @@ declare <4 x float> @llvm.x86.sse41.round.ps(<4 x float>, i32) nounwind readnone define <2 x double> @test_x86_sse41_round_sd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_sse41_round_sd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vroundsd $7, %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %a0, <2 x double> %a1, i32 7) ; <<2 x double>> [#uses=1] @@ -1468,7 +1403,7 @@ declare <2 x double> @llvm.x86.sse41.round.sd(<2 x double>, <2 x double>, i32) n define <4 x float> @test_x86_sse41_round_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse41_round_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vroundss $7, %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %a0, <4 x float> %a1, i32 7) ; <<4 x float>> [#uses=1] @@ -1479,7 +1414,7 @@ declare <4 x float> @llvm.x86.sse41.round.ss(<4 x float>, <4 x float>, i32) noun define i32 @test_x86_sse42_pcmpestri128(<16 x i8> %a0, <16 x i8> %a2) { ; CHECK-LABEL: test_x86_sse42_pcmpestri128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl $7, %eax ; CHECK-NEXT: movl $7, %edx ; CHECK-NEXT: vpcmpestri $7, %xmm1, %xmm0 @@ -1493,7 +1428,7 @@ declare i32 @llvm.x86.sse42.pcmpestri128(<16 x i8>, i32, <16 x i8>, i32, i8) nou define i32 @test_x86_sse42_pcmpestri128_load(<16 x i8>* %a0, <16 x i8>* %a2) { ; CHECK-LABEL: test_x86_sse42_pcmpestri128_load: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vmovdqa (%eax), %xmm0 @@ -1511,7 +1446,7 @@ define i32 @test_x86_sse42_pcmpestri128_load(<16 x i8>* %a0, <16 x i8>* %a2) { define i32 @test_x86_sse42_pcmpestria128(<16 x i8> %a0, <16 x i8> %a2) { ; CHECK-LABEL: test_x86_sse42_pcmpestria128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl $7, %eax ; CHECK-NEXT: movl $7, %edx ; CHECK-NEXT: vpcmpestri $7, %xmm1, %xmm0 @@ -1526,7 +1461,7 @@ declare i32 @llvm.x86.sse42.pcmpestria128(<16 x i8>, i32, <16 x i8>, i32, i8) no define i32 @test_x86_sse42_pcmpestric128(<16 x i8> %a0, <16 x i8> %a2) { ; CHECK-LABEL: test_x86_sse42_pcmpestric128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl $7, %eax ; CHECK-NEXT: movl $7, %edx ; CHECK-NEXT: vpcmpestri $7, %xmm1, %xmm0 @@ -1541,7 +1476,7 @@ declare i32 @llvm.x86.sse42.pcmpestric128(<16 x i8>, i32, <16 x i8>, i32, i8) no define i32 @test_x86_sse42_pcmpestrio128(<16 x i8> %a0, <16 x i8> %a2) { ; CHECK-LABEL: test_x86_sse42_pcmpestrio128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl $7, %eax ; CHECK-NEXT: movl $7, %edx ; CHECK-NEXT: vpcmpestri $7, %xmm1, %xmm0 @@ -1556,7 +1491,7 @@ declare i32 @llvm.x86.sse42.pcmpestrio128(<16 x i8>, i32, <16 x i8>, i32, i8) no define i32 @test_x86_sse42_pcmpestris128(<16 x i8> %a0, <16 x i8> %a2) { ; CHECK-LABEL: test_x86_sse42_pcmpestris128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl $7, %eax ; CHECK-NEXT: movl $7, %edx ; CHECK-NEXT: vpcmpestri $7, %xmm1, %xmm0 @@ -1571,7 +1506,7 @@ declare i32 @llvm.x86.sse42.pcmpestris128(<16 x i8>, i32, <16 x i8>, i32, i8) no define i32 @test_x86_sse42_pcmpestriz128(<16 x i8> %a0, <16 x i8> %a2) { ; CHECK-LABEL: test_x86_sse42_pcmpestriz128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl $7, %eax ; CHECK-NEXT: movl $7, %edx ; CHECK-NEXT: vpcmpestri $7, %xmm1, %xmm0 @@ -1586,7 +1521,7 @@ declare i32 @llvm.x86.sse42.pcmpestriz128(<16 x i8>, i32, <16 x i8>, i32, i8) no define <16 x i8> @test_x86_sse42_pcmpestrm128(<16 x i8> %a0, <16 x i8> %a2) { ; CHECK-LABEL: test_x86_sse42_pcmpestrm128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl $7, %eax ; CHECK-NEXT: movl $7, %edx ; CHECK-NEXT: vpcmpestrm $7, %xmm1, %xmm0 @@ -1599,7 +1534,7 @@ declare <16 x i8> @llvm.x86.sse42.pcmpestrm128(<16 x i8>, i32, <16 x i8>, i32, i define <16 x i8> @test_x86_sse42_pcmpestrm128_load(<16 x i8> %a0, <16 x i8>* %a2) { ; CHECK-LABEL: test_x86_sse42_pcmpestrm128_load: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx ; CHECK-NEXT: movl $7, %eax ; CHECK-NEXT: movl $7, %edx @@ -1613,7 +1548,7 @@ define <16 x i8> @test_x86_sse42_pcmpestrm128_load(<16 x i8> %a0, <16 x i8>* %a2 define i32 @test_x86_sse42_pcmpistri128(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse42_pcmpistri128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpcmpistri $7, %xmm1, %xmm0 ; CHECK-NEXT: movl %ecx, %eax ; CHECK-NEXT: retl @@ -1625,7 +1560,7 @@ declare i32 @llvm.x86.sse42.pcmpistri128(<16 x i8>, <16 x i8>, i8) nounwind read define i32 @test_x86_sse42_pcmpistri128_load(<16 x i8>* %a0, <16 x i8>* %a1) { ; CHECK-LABEL: test_x86_sse42_pcmpistri128_load: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx ; CHECK-NEXT: vmovdqa (%ecx), %xmm0 @@ -1641,7 +1576,7 @@ define i32 @test_x86_sse42_pcmpistri128_load(<16 x i8>* %a0, <16 x i8>* %a1) { define i32 @test_x86_sse42_pcmpistria128(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse42_pcmpistria128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpcmpistri $7, %xmm1, %xmm0 ; CHECK-NEXT: seta %al ; CHECK-NEXT: movzbl %al, %eax @@ -1654,7 +1589,7 @@ declare i32 @llvm.x86.sse42.pcmpistria128(<16 x i8>, <16 x i8>, i8) nounwind rea define i32 @test_x86_sse42_pcmpistric128(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse42_pcmpistric128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpcmpistri $7, %xmm1, %xmm0 ; CHECK-NEXT: sbbl %eax, %eax ; CHECK-NEXT: andl $1, %eax @@ -1667,7 +1602,7 @@ declare i32 @llvm.x86.sse42.pcmpistric128(<16 x i8>, <16 x i8>, i8) nounwind rea define i32 @test_x86_sse42_pcmpistrio128(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse42_pcmpistrio128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpcmpistri $7, %xmm1, %xmm0 ; CHECK-NEXT: seto %al ; CHECK-NEXT: movzbl %al, %eax @@ -1680,7 +1615,7 @@ declare i32 @llvm.x86.sse42.pcmpistrio128(<16 x i8>, <16 x i8>, i8) nounwind rea define i32 @test_x86_sse42_pcmpistris128(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse42_pcmpistris128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpcmpistri $7, %xmm1, %xmm0 ; CHECK-NEXT: sets %al ; CHECK-NEXT: movzbl %al, %eax @@ -1693,7 +1628,7 @@ declare i32 @llvm.x86.sse42.pcmpistris128(<16 x i8>, <16 x i8>, i8) nounwind rea define i32 @test_x86_sse42_pcmpistriz128(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse42_pcmpistriz128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpcmpistri $7, %xmm1, %xmm0 ; CHECK-NEXT: sete %al ; CHECK-NEXT: movzbl %al, %eax @@ -1706,7 +1641,7 @@ declare i32 @llvm.x86.sse42.pcmpistriz128(<16 x i8>, <16 x i8>, i8) nounwind rea define <16 x i8> @test_x86_sse42_pcmpistrm128(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_sse42_pcmpistrm128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpcmpistrm $7, %xmm1, %xmm0 ; CHECK-NEXT: retl %res = call <16 x i8> @llvm.x86.sse42.pcmpistrm128(<16 x i8> %a0, <16 x i8> %a1, i8 7) ; <<16 x i8>> [#uses=1] @@ -1717,7 +1652,7 @@ declare <16 x i8> @llvm.x86.sse42.pcmpistrm128(<16 x i8>, <16 x i8>, i8) nounwin define <16 x i8> @test_x86_sse42_pcmpistrm128_load(<16 x i8> %a0, <16 x i8>* %a1) { ; CHECK-LABEL: test_x86_sse42_pcmpistrm128_load: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vpcmpistrm $7, (%eax), %xmm0 ; CHECK-NEXT: retl @@ -1729,7 +1664,7 @@ define <16 x i8> @test_x86_sse42_pcmpistrm128_load(<16 x i8> %a0, <16 x i8>* %a1 define <4 x float> @test_x86_sse_add_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_add_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vaddss %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse.add.ss(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1] @@ -1740,7 +1675,7 @@ declare <4 x float> @llvm.x86.sse.add.ss(<4 x float>, <4 x float>) nounwind read define <4 x float> @test_x86_sse_cmp_ps(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_cmp_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcmpordps %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse.cmp.ps(<4 x float> %a0, <4 x float> %a1, i8 7) ; <<4 x float>> [#uses=1] @@ -1751,7 +1686,7 @@ declare <4 x float> @llvm.x86.sse.cmp.ps(<4 x float>, <4 x float>, i8) nounwind define <4 x float> @test_x86_sse_cmp_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_cmp_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcmpordss %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %a0, <4 x float> %a1, i8 7) ; <<4 x float>> [#uses=1] @@ -1762,7 +1697,7 @@ declare <4 x float> @llvm.x86.sse.cmp.ss(<4 x float>, <4 x float>, i8) nounwind define i32 @test_x86_sse_comieq_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_comieq_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcomiss %xmm1, %xmm0 ; CHECK-NEXT: sete %al ; CHECK-NEXT: movzbl %al, %eax @@ -1775,7 +1710,7 @@ declare i32 @llvm.x86.sse.comieq.ss(<4 x float>, <4 x float>) nounwind readnone define i32 @test_x86_sse_comige_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_comige_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcomiss %xmm1, %xmm0 ; CHECK-NEXT: setae %al ; CHECK-NEXT: movzbl %al, %eax @@ -1788,7 +1723,7 @@ declare i32 @llvm.x86.sse.comige.ss(<4 x float>, <4 x float>) nounwind readnone define i32 @test_x86_sse_comigt_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_comigt_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcomiss %xmm1, %xmm0 ; CHECK-NEXT: seta %al ; CHECK-NEXT: movzbl %al, %eax @@ -1801,7 +1736,7 @@ declare i32 @llvm.x86.sse.comigt.ss(<4 x float>, <4 x float>) nounwind readnone define i32 @test_x86_sse_comile_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_comile_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcomiss %xmm1, %xmm0 ; CHECK-NEXT: setbe %al ; CHECK-NEXT: movzbl %al, %eax @@ -1814,7 +1749,7 @@ declare i32 @llvm.x86.sse.comile.ss(<4 x float>, <4 x float>) nounwind readnone define i32 @test_x86_sse_comilt_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_comilt_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcomiss %xmm1, %xmm0 ; CHECK-NEXT: sbbl %eax, %eax ; CHECK-NEXT: andl $1, %eax @@ -1827,7 +1762,7 @@ declare i32 @llvm.x86.sse.comilt.ss(<4 x float>, <4 x float>) nounwind readnone define i32 @test_x86_sse_comineq_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_comineq_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcomiss %xmm1, %xmm0 ; CHECK-NEXT: setne %al ; CHECK-NEXT: movzbl %al, %eax @@ -1840,7 +1775,7 @@ declare i32 @llvm.x86.sse.comineq.ss(<4 x float>, <4 x float>) nounwind readnone define <4 x float> @test_x86_sse_cvtsi2ss(<4 x float> %a0) { ; CHECK-LABEL: test_x86_sse_cvtsi2ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl $7, %eax ; CHECK-NEXT: vcvtsi2ssl %eax, %xmm0, %xmm0 ; CHECK-NEXT: retl @@ -1852,7 +1787,7 @@ declare <4 x float> @llvm.x86.sse.cvtsi2ss(<4 x float>, i32) nounwind readnone define i32 @test_x86_sse_cvtss2si(<4 x float> %a0) { ; CHECK-LABEL: test_x86_sse_cvtss2si: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvtss2si %xmm0, %eax ; CHECK-NEXT: retl %res = call i32 @llvm.x86.sse.cvtss2si(<4 x float> %a0) ; [#uses=1] @@ -1863,7 +1798,7 @@ declare i32 @llvm.x86.sse.cvtss2si(<4 x float>) nounwind readnone define i32 @test_x86_sse_cvttss2si(<4 x float> %a0) { ; CHECK-LABEL: test_x86_sse_cvttss2si: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvttss2si %xmm0, %eax ; CHECK-NEXT: retl %res = call i32 @llvm.x86.sse.cvttss2si(<4 x float> %a0) ; [#uses=1] @@ -1874,7 +1809,7 @@ declare i32 @llvm.x86.sse.cvttss2si(<4 x float>) nounwind readnone define <4 x float> @test_x86_sse_div_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_div_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vdivss %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse.div.ss(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1] @@ -1885,7 +1820,7 @@ declare <4 x float> @llvm.x86.sse.div.ss(<4 x float>, <4 x float>) nounwind read define void @test_x86_sse_ldmxcsr(i8* %a0) { ; CHECK-LABEL: test_x86_sse_ldmxcsr: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vldmxcsr (%eax) ; CHECK-NEXT: retl @@ -1898,7 +1833,7 @@ declare void @llvm.x86.sse.ldmxcsr(i8*) nounwind define <4 x float> @test_x86_sse_max_ps(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_max_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vmaxps %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1] @@ -1909,7 +1844,7 @@ declare <4 x float> @llvm.x86.sse.max.ps(<4 x float>, <4 x float>) nounwind read define <4 x float> @test_x86_sse_max_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_max_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vmaxss %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse.max.ss(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1] @@ -1920,7 +1855,7 @@ declare <4 x float> @llvm.x86.sse.max.ss(<4 x float>, <4 x float>) nounwind read define <4 x float> @test_x86_sse_min_ps(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_min_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vminps %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1] @@ -1931,7 +1866,7 @@ declare <4 x float> @llvm.x86.sse.min.ps(<4 x float>, <4 x float>) nounwind read define <4 x float> @test_x86_sse_min_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_min_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vminss %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse.min.ss(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1] @@ -1942,7 +1877,7 @@ declare <4 x float> @llvm.x86.sse.min.ss(<4 x float>, <4 x float>) nounwind read define i32 @test_x86_sse_movmsk_ps(<4 x float> %a0) { ; CHECK-LABEL: test_x86_sse_movmsk_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vmovmskps %xmm0, %eax ; CHECK-NEXT: retl %res = call i32 @llvm.x86.sse.movmsk.ps(<4 x float> %a0) ; [#uses=1] @@ -1954,7 +1889,7 @@ declare i32 @llvm.x86.sse.movmsk.ps(<4 x float>) nounwind readnone define <4 x float> @test_x86_sse_mul_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_mul_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vmulss %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse.mul.ss(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1] @@ -1965,7 +1900,7 @@ declare <4 x float> @llvm.x86.sse.mul.ss(<4 x float>, <4 x float>) nounwind read define <4 x float> @test_x86_sse_rcp_ps(<4 x float> %a0) { ; CHECK-LABEL: test_x86_sse_rcp_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vrcpps %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse.rcp.ps(<4 x float> %a0) ; <<4 x float>> [#uses=1] @@ -1976,7 +1911,7 @@ declare <4 x float> @llvm.x86.sse.rcp.ps(<4 x float>) nounwind readnone define <4 x float> @test_x86_sse_rcp_ss(<4 x float> %a0) { ; CHECK-LABEL: test_x86_sse_rcp_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vrcpss %xmm0, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> %a0) ; <<4 x float>> [#uses=1] @@ -1987,7 +1922,7 @@ declare <4 x float> @llvm.x86.sse.rcp.ss(<4 x float>) nounwind readnone define <4 x float> @test_x86_sse_rsqrt_ps(<4 x float> %a0) { ; CHECK-LABEL: test_x86_sse_rsqrt_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vrsqrtps %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse.rsqrt.ps(<4 x float> %a0) ; <<4 x float>> [#uses=1] @@ -1998,7 +1933,7 @@ declare <4 x float> @llvm.x86.sse.rsqrt.ps(<4 x float>) nounwind readnone define <4 x float> @test_x86_sse_rsqrt_ss(<4 x float> %a0) { ; CHECK-LABEL: test_x86_sse_rsqrt_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vrsqrtss %xmm0, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> %a0) ; <<4 x float>> [#uses=1] @@ -2009,7 +1944,7 @@ declare <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float>) nounwind readnone define <4 x float> @test_x86_sse_sqrt_ps(<4 x float> %a0) { ; CHECK-LABEL: test_x86_sse_sqrt_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vsqrtps %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse.sqrt.ps(<4 x float> %a0) ; <<4 x float>> [#uses=1] @@ -2020,7 +1955,7 @@ declare <4 x float> @llvm.x86.sse.sqrt.ps(<4 x float>) nounwind readnone define <4 x float> @test_x86_sse_sqrt_ss(<4 x float> %a0) { ; CHECK-LABEL: test_x86_sse_sqrt_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vsqrtss %xmm0, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float> %a0) ; <<4 x float>> [#uses=1] @@ -2031,7 +1966,7 @@ declare <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float>) nounwind readnone define void @test_x86_sse_stmxcsr(i8* %a0) { ; CHECK-LABEL: test_x86_sse_stmxcsr: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vstmxcsr (%eax) ; CHECK-NEXT: retl @@ -2043,7 +1978,7 @@ declare void @llvm.x86.sse.stmxcsr(i8*) nounwind define void @test_x86_sse_storeu_ps(i8* %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_storeu_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vmovups %xmm0, (%eax) ; CHECK-NEXT: retl @@ -2055,7 +1990,7 @@ declare void @llvm.x86.sse.storeu.ps(i8*, <4 x float>) nounwind define <4 x float> @test_x86_sse_sub_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_sub_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vsubss %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.sse.sub.ss(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1] @@ -2066,7 +2001,7 @@ declare <4 x float> @llvm.x86.sse.sub.ss(<4 x float>, <4 x float>) nounwind read define i32 @test_x86_sse_ucomieq_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_ucomieq_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vucomiss %xmm1, %xmm0 ; CHECK-NEXT: sete %al ; CHECK-NEXT: movzbl %al, %eax @@ -2079,7 +2014,7 @@ declare i32 @llvm.x86.sse.ucomieq.ss(<4 x float>, <4 x float>) nounwind readnone define i32 @test_x86_sse_ucomige_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_ucomige_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vucomiss %xmm1, %xmm0 ; CHECK-NEXT: setae %al ; CHECK-NEXT: movzbl %al, %eax @@ -2092,7 +2027,7 @@ declare i32 @llvm.x86.sse.ucomige.ss(<4 x float>, <4 x float>) nounwind readnone define i32 @test_x86_sse_ucomigt_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_ucomigt_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vucomiss %xmm1, %xmm0 ; CHECK-NEXT: seta %al ; CHECK-NEXT: movzbl %al, %eax @@ -2105,7 +2040,7 @@ declare i32 @llvm.x86.sse.ucomigt.ss(<4 x float>, <4 x float>) nounwind readnone define i32 @test_x86_sse_ucomile_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_ucomile_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vucomiss %xmm1, %xmm0 ; CHECK-NEXT: setbe %al ; CHECK-NEXT: movzbl %al, %eax @@ -2118,7 +2053,7 @@ declare i32 @llvm.x86.sse.ucomile.ss(<4 x float>, <4 x float>) nounwind readnone define i32 @test_x86_sse_ucomilt_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_ucomilt_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vucomiss %xmm1, %xmm0 ; CHECK-NEXT: sbbl %eax, %eax ; CHECK-NEXT: andl $1, %eax @@ -2131,7 +2066,7 @@ declare i32 @llvm.x86.sse.ucomilt.ss(<4 x float>, <4 x float>) nounwind readnone define i32 @test_x86_sse_ucomineq_ss(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_sse_ucomineq_ss: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vucomiss %xmm1, %xmm0 ; CHECK-NEXT: setne %al ; CHECK-NEXT: movzbl %al, %eax @@ -2144,7 +2079,7 @@ declare i32 @llvm.x86.sse.ucomineq.ss(<4 x float>, <4 x float>) nounwind readnon define <16 x i8> @test_x86_ssse3_pabs_b_128(<16 x i8> %a0) { ; CHECK-LABEL: test_x86_ssse3_pabs_b_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpabsb %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <16 x i8> @llvm.x86.ssse3.pabs.b.128(<16 x i8> %a0) ; <<16 x i8>> [#uses=1] @@ -2155,7 +2090,7 @@ declare <16 x i8> @llvm.x86.ssse3.pabs.b.128(<16 x i8>) nounwind readnone define <4 x i32> @test_x86_ssse3_pabs_d_128(<4 x i32> %a0) { ; CHECK-LABEL: test_x86_ssse3_pabs_d_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpabsd %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.ssse3.pabs.d.128(<4 x i32> %a0) ; <<4 x i32>> [#uses=1] @@ -2166,7 +2101,7 @@ declare <4 x i32> @llvm.x86.ssse3.pabs.d.128(<4 x i32>) nounwind readnone define <8 x i16> @test_x86_ssse3_pabs_w_128(<8 x i16> %a0) { ; CHECK-LABEL: test_x86_ssse3_pabs_w_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpabsw %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.ssse3.pabs.w.128(<8 x i16> %a0) ; <<8 x i16>> [#uses=1] @@ -2177,7 +2112,7 @@ declare <8 x i16> @llvm.x86.ssse3.pabs.w.128(<8 x i16>) nounwind readnone define <4 x i32> @test_x86_ssse3_phadd_d_128(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: test_x86_ssse3_phadd_d_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vphaddd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.ssse3.phadd.d.128(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1] @@ -2188,7 +2123,7 @@ declare <4 x i32> @llvm.x86.ssse3.phadd.d.128(<4 x i32>, <4 x i32>) nounwind rea define <8 x i16> @test_x86_ssse3_phadd_sw_128(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_ssse3_phadd_sw_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vphaddsw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.ssse3.phadd.sw.128(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -2199,7 +2134,7 @@ declare <8 x i16> @llvm.x86.ssse3.phadd.sw.128(<8 x i16>, <8 x i16>) nounwind re define <8 x i16> @test_x86_ssse3_phadd_w_128(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_ssse3_phadd_w_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vphaddw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.ssse3.phadd.w.128(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -2210,7 +2145,7 @@ declare <8 x i16> @llvm.x86.ssse3.phadd.w.128(<8 x i16>, <8 x i16>) nounwind rea define <4 x i32> @test_x86_ssse3_phsub_d_128(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: test_x86_ssse3_phsub_d_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vphsubd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.ssse3.phsub.d.128(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1] @@ -2221,7 +2156,7 @@ declare <4 x i32> @llvm.x86.ssse3.phsub.d.128(<4 x i32>, <4 x i32>) nounwind rea define <8 x i16> @test_x86_ssse3_phsub_sw_128(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_ssse3_phsub_sw_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vphsubsw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.ssse3.phsub.sw.128(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -2232,7 +2167,7 @@ declare <8 x i16> @llvm.x86.ssse3.phsub.sw.128(<8 x i16>, <8 x i16>) nounwind re define <8 x i16> @test_x86_ssse3_phsub_w_128(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_ssse3_phsub_w_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vphsubw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.ssse3.phsub.w.128(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -2243,7 +2178,7 @@ declare <8 x i16> @llvm.x86.ssse3.phsub.w.128(<8 x i16>, <8 x i16>) nounwind rea define <8 x i16> @test_x86_ssse3_pmadd_ub_sw_128(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_ssse3_pmadd_ub_sw_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a0, <16 x i8> %a1) ; <<8 x i16>> [#uses=1] @@ -2254,7 +2189,7 @@ declare <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8>, <16 x i8>) nounwind define <8 x i16> @test_x86_ssse3_pmul_hr_sw_128(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_ssse3_pmul_hr_sw_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpmulhrsw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.ssse3.pmul.hr.sw.128(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -2265,7 +2200,7 @@ declare <8 x i16> @llvm.x86.ssse3.pmul.hr.sw.128(<8 x i16>, <8 x i16>) nounwind define <16 x i8> @test_x86_ssse3_pshuf_b_128(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_ssse3_pshuf_b_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpshufb %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %a0, <16 x i8> %a1) ; <<16 x i8>> [#uses=1] @@ -2276,7 +2211,7 @@ declare <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8>, <16 x i8>) nounwind rea define <16 x i8> @test_x86_ssse3_psign_b_128(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK-LABEL: test_x86_ssse3_psign_b_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsignb %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <16 x i8> @llvm.x86.ssse3.psign.b.128(<16 x i8> %a0, <16 x i8> %a1) ; <<16 x i8>> [#uses=1] @@ -2287,7 +2222,7 @@ declare <16 x i8> @llvm.x86.ssse3.psign.b.128(<16 x i8>, <16 x i8>) nounwind rea define <4 x i32> @test_x86_ssse3_psign_d_128(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK-LABEL: test_x86_ssse3_psign_d_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsignd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x i32> @llvm.x86.ssse3.psign.d.128(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1] @@ -2298,7 +2233,7 @@ declare <4 x i32> @llvm.x86.ssse3.psign.d.128(<4 x i32>, <4 x i32>) nounwind rea define <8 x i16> @test_x86_ssse3_psign_w_128(<8 x i16> %a0, <8 x i16> %a1) { ; CHECK-LABEL: test_x86_ssse3_psign_w_128: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpsignw %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <8 x i16> @llvm.x86.ssse3.psign.w.128(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1] @@ -2309,7 +2244,7 @@ declare <8 x i16> @llvm.x86.ssse3.psign.w.128(<8 x i16>, <8 x i16>) nounwind rea define <4 x double> @test_x86_avx_addsub_pd_256(<4 x double> %a0, <4 x double> %a1) { ; CHECK-LABEL: test_x86_avx_addsub_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vaddsubpd %ymm1, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double> %a0, <4 x double> %a1) ; <<4 x double>> [#uses=1] @@ -2320,7 +2255,7 @@ declare <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double>, <4 x double>) nou define <8 x float> @test_x86_avx_addsub_ps_256(<8 x float> %a0, <8 x float> %a1) { ; CHECK-LABEL: test_x86_avx_addsub_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vaddsubps %ymm1, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float> %a0, <8 x float> %a1) ; <<8 x float>> [#uses=1] @@ -2331,7 +2266,7 @@ declare <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float>, <8 x float>) nounwi define <4 x double> @test_x86_avx_blendv_pd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) { ; CHECK-LABEL: test_x86_avx_blendv_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) ; <<4 x double>> [#uses=1] @@ -2342,7 +2277,7 @@ declare <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double>, <4 x double>, <4 define <8 x float> @test_x86_avx_blendv_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) { ; CHECK-LABEL: test_x86_avx_blendv_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vblendvps %ymm2, %ymm1, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) ; <<8 x float>> [#uses=1] @@ -2353,7 +2288,7 @@ declare <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float>, <8 x float>, <8 x f define <4 x double> @test_x86_avx_cmp_pd_256(<4 x double> %a0, <4 x double> %a1) { ; CHECK-LABEL: test_x86_avx_cmp_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcmpordpd %ymm1, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <4 x double> @llvm.x86.avx.cmp.pd.256(<4 x double> %a0, <4 x double> %a1, i8 7) ; <<4 x double>> [#uses=1] @@ -2364,7 +2299,7 @@ declare <4 x double> @llvm.x86.avx.cmp.pd.256(<4 x double>, <4 x double>, i8) no define <8 x float> @test_x86_avx_cmp_ps_256(<8 x float> %a0, <8 x float> %a1) { ; CHECK-LABEL: test_x86_avx_cmp_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcmpordps %ymm1, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float> %a0, <8 x float> %a1, i8 7) ; <<8 x float>> [#uses=1] @@ -2373,7 +2308,7 @@ define <8 x float> @test_x86_avx_cmp_ps_256(<8 x float> %a0, <8 x float> %a1) { define <8 x float> @test_x86_avx_cmp_ps_256_pseudo_op(<8 x float> %a0, <8 x float> %a1) { ; CHECK-LABEL: test_x86_avx_cmp_ps_256_pseudo_op: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcmpeqps %ymm1, %ymm0, %ymm1 ; CHECK-NEXT: vcmpltps %ymm1, %ymm0, %ymm1 ; CHECK-NEXT: vcmpleps %ymm1, %ymm0, %ymm1 @@ -2446,7 +2381,7 @@ declare <8 x float> @llvm.x86.avx.cmp.ps.256(<8 x float>, <8 x float>, i8) nounw define <4 x float> @test_x86_avx_cvt_pd2_ps_256(<4 x double> %a0) { ; CHECK-LABEL: test_x86_avx_cvt_pd2_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvtpd2psy %ymm0, %xmm0 ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retl @@ -2458,7 +2393,7 @@ declare <4 x float> @llvm.x86.avx.cvt.pd2.ps.256(<4 x double>) nounwind readnone define <4 x i32> @test_x86_avx_cvt_pd2dq_256(<4 x double> %a0) { ; CHECK-LABEL: test_x86_avx_cvt_pd2dq_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvtpd2dqy %ymm0, %xmm0 ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retl @@ -2470,7 +2405,7 @@ declare <4 x i32> @llvm.x86.avx.cvt.pd2dq.256(<4 x double>) nounwind readnone define <4 x double> @test_x86_avx_cvt_ps2_pd_256(<4 x float> %a0) { ; CHECK-LABEL: test_x86_avx_cvt_ps2_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvtps2pd %xmm0, %ymm0 ; CHECK-NEXT: retl %res = call <4 x double> @llvm.x86.avx.cvt.ps2.pd.256(<4 x float> %a0) ; <<4 x double>> [#uses=1] @@ -2481,7 +2416,7 @@ declare <4 x double> @llvm.x86.avx.cvt.ps2.pd.256(<4 x float>) nounwind readnone define <8 x i32> @test_x86_avx_cvt_ps2dq_256(<8 x float> %a0) { ; CHECK-LABEL: test_x86_avx_cvt_ps2dq_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvtps2dq %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <8 x i32> @llvm.x86.avx.cvt.ps2dq.256(<8 x float> %a0) ; <<8 x i32>> [#uses=1] @@ -2492,7 +2427,7 @@ declare <8 x i32> @llvm.x86.avx.cvt.ps2dq.256(<8 x float>) nounwind readnone define <4 x double> @test_x86_avx_cvtdq2_pd_256(<4 x i32> %a0) { ; CHECK-LABEL: test_x86_avx_cvtdq2_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvtdq2pd %xmm0, %ymm0 ; CHECK-NEXT: retl %res = call <4 x double> @llvm.x86.avx.cvtdq2.pd.256(<4 x i32> %a0) ; <<4 x double>> [#uses=1] @@ -2503,7 +2438,7 @@ declare <4 x double> @llvm.x86.avx.cvtdq2.pd.256(<4 x i32>) nounwind readnone define <8 x float> @test_x86_avx_cvtdq2_ps_256(<8 x i32> %a0) { ; CHECK-LABEL: test_x86_avx_cvtdq2_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvtdq2ps %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <8 x float> @llvm.x86.avx.cvtdq2.ps.256(<8 x i32> %a0) ; <<8 x float>> [#uses=1] @@ -2514,7 +2449,7 @@ declare <8 x float> @llvm.x86.avx.cvtdq2.ps.256(<8 x i32>) nounwind readnone define <4 x i32> @test_x86_avx_cvtt_pd2dq_256(<4 x double> %a0) { ; CHECK-LABEL: test_x86_avx_cvtt_pd2dq_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvttpd2dqy %ymm0, %xmm0 ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retl @@ -2526,7 +2461,7 @@ declare <4 x i32> @llvm.x86.avx.cvtt.pd2dq.256(<4 x double>) nounwind readnone define <8 x i32> @test_x86_avx_cvtt_ps2dq_256(<8 x float> %a0) { ; CHECK-LABEL: test_x86_avx_cvtt_ps2dq_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vcvttps2dq %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <8 x i32> @llvm.x86.avx.cvtt.ps2dq.256(<8 x float> %a0) ; <<8 x i32>> [#uses=1] @@ -2537,7 +2472,7 @@ declare <8 x i32> @llvm.x86.avx.cvtt.ps2dq.256(<8 x float>) nounwind readnone define <8 x float> @test_x86_avx_dp_ps_256(<8 x float> %a0, <8 x float> %a1) { ; CHECK-LABEL: test_x86_avx_dp_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vdpps $7, %ymm1, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <8 x float> @llvm.x86.avx.dp.ps.256(<8 x float> %a0, <8 x float> %a1, i8 7) ; <<8 x float>> [#uses=1] @@ -2548,7 +2483,7 @@ declare <8 x float> @llvm.x86.avx.dp.ps.256(<8 x float>, <8 x float>, i8) nounwi define <4 x double> @test_x86_avx_hadd_pd_256(<4 x double> %a0, <4 x double> %a1) { ; CHECK-LABEL: test_x86_avx_hadd_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vhaddpd %ymm1, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <4 x double> @llvm.x86.avx.hadd.pd.256(<4 x double> %a0, <4 x double> %a1) ; <<4 x double>> [#uses=1] @@ -2559,7 +2494,7 @@ declare <4 x double> @llvm.x86.avx.hadd.pd.256(<4 x double>, <4 x double>) nounw define <8 x float> @test_x86_avx_hadd_ps_256(<8 x float> %a0, <8 x float> %a1) { ; CHECK-LABEL: test_x86_avx_hadd_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vhaddps %ymm1, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <8 x float> @llvm.x86.avx.hadd.ps.256(<8 x float> %a0, <8 x float> %a1) ; <<8 x float>> [#uses=1] @@ -2570,7 +2505,7 @@ declare <8 x float> @llvm.x86.avx.hadd.ps.256(<8 x float>, <8 x float>) nounwind define <4 x double> @test_x86_avx_hsub_pd_256(<4 x double> %a0, <4 x double> %a1) { ; CHECK-LABEL: test_x86_avx_hsub_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vhsubpd %ymm1, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <4 x double> @llvm.x86.avx.hsub.pd.256(<4 x double> %a0, <4 x double> %a1) ; <<4 x double>> [#uses=1] @@ -2581,7 +2516,7 @@ declare <4 x double> @llvm.x86.avx.hsub.pd.256(<4 x double>, <4 x double>) nounw define <8 x float> @test_x86_avx_hsub_ps_256(<8 x float> %a0, <8 x float> %a1) { ; CHECK-LABEL: test_x86_avx_hsub_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vhsubps %ymm1, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <8 x float> @llvm.x86.avx.hsub.ps.256(<8 x float> %a0, <8 x float> %a1) ; <<8 x float>> [#uses=1] @@ -2592,7 +2527,7 @@ declare <8 x float> @llvm.x86.avx.hsub.ps.256(<8 x float>, <8 x float>) nounwind define <32 x i8> @test_x86_avx_ldu_dq_256(i8* %a0) { ; CHECK-LABEL: test_x86_avx_ldu_dq_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vlddqu (%eax), %ymm0 ; CHECK-NEXT: retl @@ -2602,107 +2537,107 @@ define <32 x i8> @test_x86_avx_ldu_dq_256(i8* %a0) { declare <32 x i8> @llvm.x86.avx.ldu.dq.256(i8*) nounwind readonly -define <2 x double> @test_x86_avx_maskload_pd(i8* %a0, <2 x double> %a1) { +define <2 x double> @test_x86_avx_maskload_pd(i8* %a0, <2 x i64> %mask) { ; CHECK-LABEL: test_x86_avx_maskload_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vmaskmovpd (%eax), %xmm0, %xmm0 ; CHECK-NEXT: retl - %res = call <2 x double> @llvm.x86.avx.maskload.pd(i8* %a0, <2 x double> %a1) ; <<2 x double>> [#uses=1] + %res = call <2 x double> @llvm.x86.avx.maskload.pd(i8* %a0, <2 x i64> %mask) ; <<2 x double>> [#uses=1] ret <2 x double> %res } -declare <2 x double> @llvm.x86.avx.maskload.pd(i8*, <2 x double>) nounwind readonly +declare <2 x double> @llvm.x86.avx.maskload.pd(i8*, <2 x i64>) nounwind readonly -define <4 x double> @test_x86_avx_maskload_pd_256(i8* %a0, <4 x double> %a1) { +define <4 x double> @test_x86_avx_maskload_pd_256(i8* %a0, <4 x i64> %mask) { ; CHECK-LABEL: test_x86_avx_maskload_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vmaskmovpd (%eax), %ymm0, %ymm0 ; CHECK-NEXT: retl - %res = call <4 x double> @llvm.x86.avx.maskload.pd.256(i8* %a0, <4 x double> %a1) ; <<4 x double>> [#uses=1] + %res = call <4 x double> @llvm.x86.avx.maskload.pd.256(i8* %a0, <4 x i64> %mask) ; <<4 x double>> [#uses=1] ret <4 x double> %res } -declare <4 x double> @llvm.x86.avx.maskload.pd.256(i8*, <4 x double>) nounwind readonly +declare <4 x double> @llvm.x86.avx.maskload.pd.256(i8*, <4 x i64>) nounwind readonly -define <4 x float> @test_x86_avx_maskload_ps(i8* %a0, <4 x float> %a1) { +define <4 x float> @test_x86_avx_maskload_ps(i8* %a0, <4 x i32> %mask) { ; CHECK-LABEL: test_x86_avx_maskload_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vmaskmovps (%eax), %xmm0, %xmm0 ; CHECK-NEXT: retl - %res = call <4 x float> @llvm.x86.avx.maskload.ps(i8* %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1] + %res = call <4 x float> @llvm.x86.avx.maskload.ps(i8* %a0, <4 x i32> %mask) ; <<4 x float>> [#uses=1] ret <4 x float> %res } -declare <4 x float> @llvm.x86.avx.maskload.ps(i8*, <4 x float>) nounwind readonly +declare <4 x float> @llvm.x86.avx.maskload.ps(i8*, <4 x i32>) nounwind readonly -define <8 x float> @test_x86_avx_maskload_ps_256(i8* %a0, <8 x float> %a1) { +define <8 x float> @test_x86_avx_maskload_ps_256(i8* %a0, <8 x i32> %mask) { ; CHECK-LABEL: test_x86_avx_maskload_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vmaskmovps (%eax), %ymm0, %ymm0 ; CHECK-NEXT: retl - %res = call <8 x float> @llvm.x86.avx.maskload.ps.256(i8* %a0, <8 x float> %a1) ; <<8 x float>> [#uses=1] + %res = call <8 x float> @llvm.x86.avx.maskload.ps.256(i8* %a0, <8 x i32> %mask) ; <<8 x float>> [#uses=1] ret <8 x float> %res } -declare <8 x float> @llvm.x86.avx.maskload.ps.256(i8*, <8 x float>) nounwind readonly +declare <8 x float> @llvm.x86.avx.maskload.ps.256(i8*, <8 x i32>) nounwind readonly -define void @test_x86_avx_maskstore_pd(i8* %a0, <2 x double> %a1, <2 x double> %a2) { +define void @test_x86_avx_maskstore_pd(i8* %a0, <2 x i64> %mask, <2 x double> %a2) { ; CHECK-LABEL: test_x86_avx_maskstore_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vmaskmovpd %xmm1, %xmm0, (%eax) ; CHECK-NEXT: retl - call void @llvm.x86.avx.maskstore.pd(i8* %a0, <2 x double> %a1, <2 x double> %a2) + call void @llvm.x86.avx.maskstore.pd(i8* %a0, <2 x i64> %mask, <2 x double> %a2) ret void } -declare void @llvm.x86.avx.maskstore.pd(i8*, <2 x double>, <2 x double>) nounwind +declare void @llvm.x86.avx.maskstore.pd(i8*, <2 x i64>, <2 x double>) nounwind -define void @test_x86_avx_maskstore_pd_256(i8* %a0, <4 x double> %a1, <4 x double> %a2) { +define void @test_x86_avx_maskstore_pd_256(i8* %a0, <4 x i64> %mask, <4 x double> %a2) { ; CHECK-LABEL: test_x86_avx_maskstore_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vmaskmovpd %ymm1, %ymm0, (%eax) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retl - call void @llvm.x86.avx.maskstore.pd.256(i8* %a0, <4 x double> %a1, <4 x double> %a2) + call void @llvm.x86.avx.maskstore.pd.256(i8* %a0, <4 x i64> %mask, <4 x double> %a2) ret void } -declare void @llvm.x86.avx.maskstore.pd.256(i8*, <4 x double>, <4 x double>) nounwind +declare void @llvm.x86.avx.maskstore.pd.256(i8*, <4 x i64>, <4 x double>) nounwind -define void @test_x86_avx_maskstore_ps(i8* %a0, <4 x float> %a1, <4 x float> %a2) { +define void @test_x86_avx_maskstore_ps(i8* %a0, <4 x i32> %mask, <4 x float> %a2) { ; CHECK-LABEL: test_x86_avx_maskstore_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vmaskmovps %xmm1, %xmm0, (%eax) ; CHECK-NEXT: retl - call void @llvm.x86.avx.maskstore.ps(i8* %a0, <4 x float> %a1, <4 x float> %a2) + call void @llvm.x86.avx.maskstore.ps(i8* %a0, <4 x i32> %mask, <4 x float> %a2) ret void } -declare void @llvm.x86.avx.maskstore.ps(i8*, <4 x float>, <4 x float>) nounwind +declare void @llvm.x86.avx.maskstore.ps(i8*, <4 x i32>, <4 x float>) nounwind -define void @test_x86_avx_maskstore_ps_256(i8* %a0, <8 x float> %a1, <8 x float> %a2) { +define void @test_x86_avx_maskstore_ps_256(i8* %a0, <8 x i32> %mask, <8 x float> %a2) { ; CHECK-LABEL: test_x86_avx_maskstore_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vmaskmovps %ymm1, %ymm0, (%eax) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retl - call void @llvm.x86.avx.maskstore.ps.256(i8* %a0, <8 x float> %a1, <8 x float> %a2) + call void @llvm.x86.avx.maskstore.ps.256(i8* %a0, <8 x i32> %mask, <8 x float> %a2) ret void } -declare void @llvm.x86.avx.maskstore.ps.256(i8*, <8 x float>, <8 x float>) nounwind +declare void @llvm.x86.avx.maskstore.ps.256(i8*, <8 x i32>, <8 x float>) nounwind define <4 x double> @test_x86_avx_max_pd_256(<4 x double> %a0, <4 x double> %a1) { ; CHECK-LABEL: test_x86_avx_max_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vmaxpd %ymm1, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <4 x double> @llvm.x86.avx.max.pd.256(<4 x double> %a0, <4 x double> %a1) ; <<4 x double>> [#uses=1] @@ -2713,7 +2648,7 @@ declare <4 x double> @llvm.x86.avx.max.pd.256(<4 x double>, <4 x double>) nounwi define <8 x float> @test_x86_avx_max_ps_256(<8 x float> %a0, <8 x float> %a1) { ; CHECK-LABEL: test_x86_avx_max_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vmaxps %ymm1, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <8 x float> @llvm.x86.avx.max.ps.256(<8 x float> %a0, <8 x float> %a1) ; <<8 x float>> [#uses=1] @@ -2724,7 +2659,7 @@ declare <8 x float> @llvm.x86.avx.max.ps.256(<8 x float>, <8 x float>) nounwind define <4 x double> @test_x86_avx_min_pd_256(<4 x double> %a0, <4 x double> %a1) { ; CHECK-LABEL: test_x86_avx_min_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vminpd %ymm1, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <4 x double> @llvm.x86.avx.min.pd.256(<4 x double> %a0, <4 x double> %a1) ; <<4 x double>> [#uses=1] @@ -2735,7 +2670,7 @@ declare <4 x double> @llvm.x86.avx.min.pd.256(<4 x double>, <4 x double>) nounwi define <8 x float> @test_x86_avx_min_ps_256(<8 x float> %a0, <8 x float> %a1) { ; CHECK-LABEL: test_x86_avx_min_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vminps %ymm1, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <8 x float> @llvm.x86.avx.min.ps.256(<8 x float> %a0, <8 x float> %a1) ; <<8 x float>> [#uses=1] @@ -2746,7 +2681,7 @@ declare <8 x float> @llvm.x86.avx.min.ps.256(<8 x float>, <8 x float>) nounwind define i32 @test_x86_avx_movmsk_pd_256(<4 x double> %a0) { ; CHECK-LABEL: test_x86_avx_movmsk_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vmovmskpd %ymm0, %eax ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retl @@ -2758,7 +2693,7 @@ declare i32 @llvm.x86.avx.movmsk.pd.256(<4 x double>) nounwind readnone define i32 @test_x86_avx_movmsk_ps_256(<8 x float> %a0) { ; CHECK-LABEL: test_x86_avx_movmsk_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vmovmskps %ymm0, %eax ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retl @@ -2775,7 +2710,7 @@ declare i32 @llvm.x86.avx.movmsk.ps.256(<8 x float>) nounwind readnone define i32 @test_x86_avx_ptestc_256(<4 x i64> %a0, <4 x i64> %a1) { ; CHECK-LABEL: test_x86_avx_ptestc_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vptest %ymm1, %ymm0 ; CHECK-NEXT: sbbl %eax, %eax ; CHECK-NEXT: andl $1, %eax @@ -2789,7 +2724,7 @@ declare i32 @llvm.x86.avx.ptestc.256(<4 x i64>, <4 x i64>) nounwind readnone define i32 @test_x86_avx_ptestnzc_256(<4 x i64> %a0, <4 x i64> %a1) { ; CHECK-LABEL: test_x86_avx_ptestnzc_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vptest %ymm1, %ymm0 ; CHECK-NEXT: seta %al ; CHECK-NEXT: movzbl %al, %eax @@ -2803,7 +2738,7 @@ declare i32 @llvm.x86.avx.ptestnzc.256(<4 x i64>, <4 x i64>) nounwind readnone define i32 @test_x86_avx_ptestz_256(<4 x i64> %a0, <4 x i64> %a1) { ; CHECK-LABEL: test_x86_avx_ptestz_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vptest %ymm1, %ymm0 ; CHECK-NEXT: sete %al ; CHECK-NEXT: movzbl %al, %eax @@ -2817,7 +2752,7 @@ declare i32 @llvm.x86.avx.ptestz.256(<4 x i64>, <4 x i64>) nounwind readnone define <8 x float> @test_x86_avx_rcp_ps_256(<8 x float> %a0) { ; CHECK-LABEL: test_x86_avx_rcp_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vrcpps %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <8 x float> @llvm.x86.avx.rcp.ps.256(<8 x float> %a0) ; <<8 x float>> [#uses=1] @@ -2828,7 +2763,7 @@ declare <8 x float> @llvm.x86.avx.rcp.ps.256(<8 x float>) nounwind readnone define <4 x double> @test_x86_avx_round_pd_256(<4 x double> %a0) { ; CHECK-LABEL: test_x86_avx_round_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vroundpd $7, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <4 x double> @llvm.x86.avx.round.pd.256(<4 x double> %a0, i32 7) ; <<4 x double>> [#uses=1] @@ -2839,7 +2774,7 @@ declare <4 x double> @llvm.x86.avx.round.pd.256(<4 x double>, i32) nounwind read define <8 x float> @test_x86_avx_round_ps_256(<8 x float> %a0) { ; CHECK-LABEL: test_x86_avx_round_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vroundps $7, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <8 x float> @llvm.x86.avx.round.ps.256(<8 x float> %a0, i32 7) ; <<8 x float>> [#uses=1] @@ -2850,7 +2785,7 @@ declare <8 x float> @llvm.x86.avx.round.ps.256(<8 x float>, i32) nounwind readno define <8 x float> @test_x86_avx_rsqrt_ps_256(<8 x float> %a0) { ; CHECK-LABEL: test_x86_avx_rsqrt_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vrsqrtps %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <8 x float> @llvm.x86.avx.rsqrt.ps.256(<8 x float> %a0) ; <<8 x float>> [#uses=1] @@ -2861,7 +2796,7 @@ declare <8 x float> @llvm.x86.avx.rsqrt.ps.256(<8 x float>) nounwind readnone define <4 x double> @test_x86_avx_sqrt_pd_256(<4 x double> %a0) { ; CHECK-LABEL: test_x86_avx_sqrt_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vsqrtpd %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <4 x double> @llvm.x86.avx.sqrt.pd.256(<4 x double> %a0) ; <<4 x double>> [#uses=1] @@ -2872,7 +2807,7 @@ declare <4 x double> @llvm.x86.avx.sqrt.pd.256(<4 x double>) nounwind readnone define <8 x float> @test_x86_avx_sqrt_ps_256(<8 x float> %a0) { ; CHECK-LABEL: test_x86_avx_sqrt_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vsqrtps %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <8 x float> @llvm.x86.avx.sqrt.ps.256(<8 x float> %a0) ; <<8 x float>> [#uses=1] @@ -2885,7 +2820,7 @@ define void @test_x86_avx_storeu_dq_256(i8* %a0, <32 x i8> %a1) { ; FIXME: unfortunately the execution domain fix pass changes this to vmovups and its hard to force with no 256-bit integer instructions ; add operation forces the execution domain. ; CHECK-LABEL: test_x86_avx_storeu_dq_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm1 ; CHECK-NEXT: vmovdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] @@ -2905,7 +2840,7 @@ declare void @llvm.x86.avx.storeu.dq.256(i8*, <32 x i8>) nounwind define void @test_x86_avx_storeu_pd_256(i8* %a0, <4 x double> %a1) { ; add operation forces the execution domain. ; CHECK-LABEL: test_x86_avx_storeu_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vxorpd %ymm1, %ymm1, %ymm1 ; CHECK-NEXT: vaddpd %ymm1, %ymm0, %ymm0 @@ -2921,7 +2856,7 @@ declare void @llvm.x86.avx.storeu.pd.256(i8*, <4 x double>) nounwind define void @test_x86_avx_storeu_ps_256(i8* %a0, <8 x float> %a1) { ; CHECK-LABEL: test_x86_avx_storeu_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vmovups %ymm0, (%eax) ; CHECK-NEXT: vzeroupper @@ -2934,7 +2869,7 @@ declare void @llvm.x86.avx.storeu.ps.256(i8*, <8 x float>) nounwind define <4 x double> @test_x86_avx_vbroadcastf128_pd_256(i8* %a0) { ; CHECK-LABEL: test_x86_avx_vbroadcastf128_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vbroadcastf128 (%eax), %ymm0 ; CHECK-NEXT: retl @@ -2946,7 +2881,7 @@ declare <4 x double> @llvm.x86.avx.vbroadcastf128.pd.256(i8*) nounwind readonly define <8 x float> @test_x86_avx_vbroadcastf128_ps_256(i8* %a0) { ; CHECK-LABEL: test_x86_avx_vbroadcastf128_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vbroadcastf128 (%eax), %ymm0 ; CHECK-NEXT: retl @@ -2958,7 +2893,7 @@ declare <8 x float> @llvm.x86.avx.vbroadcastf128.ps.256(i8*) nounwind readonly define <4 x double> @test_x86_avx_vperm2f128_pd_256(<4 x double> %a0, <4 x double> %a1) { ; CHECK-LABEL: test_x86_avx_vperm2f128_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[0,1] ; CHECK-NEXT: retl %res = call <4 x double> @llvm.x86.avx.vperm2f128.pd.256(<4 x double> %a0, <4 x double> %a1, i8 7) ; <<4 x double>> [#uses=1] @@ -2969,7 +2904,7 @@ declare <4 x double> @llvm.x86.avx.vperm2f128.pd.256(<4 x double>, <4 x double>, define <8 x float> @test_x86_avx_vperm2f128_ps_256(<8 x float> %a0, <8 x float> %a1) { ; CHECK-LABEL: test_x86_avx_vperm2f128_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[0,1] ; CHECK-NEXT: retl %res = call <8 x float> @llvm.x86.avx.vperm2f128.ps.256(<8 x float> %a0, <8 x float> %a1, i8 7) ; <<8 x float>> [#uses=1] @@ -2980,7 +2915,7 @@ declare <8 x float> @llvm.x86.avx.vperm2f128.ps.256(<8 x float>, <8 x float>, i8 define <8 x i32> @test_x86_avx_vperm2f128_si_256(<8 x i32> %a0, <8 x i32> %a1) { ; CHECK-LABEL: test_x86_avx_vperm2f128_si_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[0,1] ; CHECK-NEXT: retl %res = call <8 x i32> @llvm.x86.avx.vperm2f128.si.256(<8 x i32> %a0, <8 x i32> %a1, i8 7) ; <<8 x i32>> [#uses=1] @@ -2991,7 +2926,7 @@ declare <8 x i32> @llvm.x86.avx.vperm2f128.si.256(<8 x i32>, <8 x i32>, i8) noun define <2 x double> @test_x86_avx_vpermil_pd(<2 x double> %a0) { ; CHECK-LABEL: test_x86_avx_vpermil_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpermilpd {{.*#+}} xmm0 = xmm0[1,0] ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.avx.vpermil.pd(<2 x double> %a0, i8 1) ; <<2 x double>> [#uses=1] @@ -3002,7 +2937,7 @@ declare <2 x double> @llvm.x86.avx.vpermil.pd(<2 x double>, i8) nounwind readnon define <4 x double> @test_x86_avx_vpermil_pd_256(<4 x double> %a0) { ; CHECK-LABEL: test_x86_avx_vpermil_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,1,3,2] ; CHECK-NEXT: retl %res = call <4 x double> @llvm.x86.avx.vpermil.pd.256(<4 x double> %a0, i8 7) ; <<4 x double>> [#uses=1] @@ -3013,7 +2948,7 @@ declare <4 x double> @llvm.x86.avx.vpermil.pd.256(<4 x double>, i8) nounwind rea define <4 x float> @test_x86_avx_vpermil_ps(<4 x float> %a0) { ; CHECK-LABEL: test_x86_avx_vpermil_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[3,1,0,0] ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.avx.vpermil.ps(<4 x float> %a0, i8 7) ; <<4 x float>> [#uses=1] @@ -3024,7 +2959,7 @@ declare <4 x float> @llvm.x86.avx.vpermil.ps(<4 x float>, i8) nounwind readnone define <8 x float> @test_x86_avx_vpermil_ps_256(<8 x float> %a0) { ; CHECK-LABEL: test_x86_avx_vpermil_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[3,1,0,0,7,5,4,4] ; CHECK-NEXT: retl %res = call <8 x float> @llvm.x86.avx.vpermil.ps.256(<8 x float> %a0, i8 7) ; <<8 x float>> [#uses=1] @@ -3035,7 +2970,7 @@ declare <8 x float> @llvm.x86.avx.vpermil.ps.256(<8 x float>, i8) nounwind readn define <2 x double> @test_x86_avx_vpermilvar_pd(<2 x double> %a0, <2 x i64> %a1) { ; CHECK-LABEL: test_x86_avx_vpermilvar_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpermilpd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %a0, <2 x i64> %a1) ; <<2 x double>> [#uses=1] @@ -3046,7 +2981,7 @@ declare <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double>, <2 x i64>) nounwi define <4 x double> @test_x86_avx_vpermilvar_pd_256(<4 x double> %a0, <4 x i64> %a1) { ; CHECK-LABEL: test_x86_avx_vpermilvar_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpermilpd %ymm1, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> %a1) ; <<4 x double>> [#uses=1] @@ -3054,10 +2989,18 @@ define <4 x double> @test_x86_avx_vpermilvar_pd_256(<4 x double> %a0, <4 x i64> } declare <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double>, <4 x i64>) nounwind readnone +define <4 x double> @test_x86_avx_vpermilvar_pd_256_2(<4 x double> %a0) { +; CHECK-LABEL: test_x86_avx_vpermilvar_pd_256_2: +; CHECK: ## BB#0: +; CHECK-NEXT: vpermilpd {{.*}}, %ymm0, %ymm0 ## ymm0 = ymm0[1,0,2,3] +; CHECK-NEXT: retl + %res = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> ) ; <<4 x double>> [#uses=1] + ret <4 x double> %res +} define <4 x float> @test_x86_avx_vpermilvar_ps(<4 x float> %a0, <4 x i32> %a1) { ; CHECK-LABEL: test_x86_avx_vpermilvar_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpermilps %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> %a1) ; <<4 x float>> [#uses=1] @@ -3065,7 +3008,7 @@ define <4 x float> @test_x86_avx_vpermilvar_ps(<4 x float> %a0, <4 x i32> %a1) { } define <4 x float> @test_x86_avx_vpermilvar_ps_load(<4 x float> %a0, <4 x i32>* %a1) { ; CHECK-LABEL: test_x86_avx_vpermilvar_ps_load: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vpermilps (%eax), %xmm0, %xmm0 ; CHECK-NEXT: retl @@ -3078,7 +3021,7 @@ declare <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float>, <4 x i32>) nounwind define <8 x float> @test_x86_avx_vpermilvar_ps_256(<8 x float> %a0, <8 x i32> %a1) { ; CHECK-LABEL: test_x86_avx_vpermilvar_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpermilps %ymm1, %ymm0, %ymm0 ; CHECK-NEXT: retl %res = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> %a1) ; <<8 x float>> [#uses=1] @@ -3089,7 +3032,7 @@ declare <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float>, <8 x i32>) noun define i32 @test_x86_avx_vtestc_pd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_avx_vtestc_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vtestpd %xmm1, %xmm0 ; CHECK-NEXT: sbbl %eax, %eax ; CHECK-NEXT: andl $1, %eax @@ -3102,7 +3045,7 @@ declare i32 @llvm.x86.avx.vtestc.pd(<2 x double>, <2 x double>) nounwind readnon define i32 @test_x86_avx_vtestc_pd_256(<4 x double> %a0, <4 x double> %a1) { ; CHECK-LABEL: test_x86_avx_vtestc_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vtestpd %ymm1, %ymm0 ; CHECK-NEXT: sbbl %eax, %eax ; CHECK-NEXT: andl $1, %eax @@ -3116,7 +3059,7 @@ declare i32 @llvm.x86.avx.vtestc.pd.256(<4 x double>, <4 x double>) nounwind rea define i32 @test_x86_avx_vtestc_ps(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_avx_vtestc_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vtestps %xmm1, %xmm0 ; CHECK-NEXT: sbbl %eax, %eax ; CHECK-NEXT: andl $1, %eax @@ -3129,7 +3072,7 @@ declare i32 @llvm.x86.avx.vtestc.ps(<4 x float>, <4 x float>) nounwind readnone define i32 @test_x86_avx_vtestc_ps_256(<8 x float> %a0, <8 x float> %a1) { ; CHECK-LABEL: test_x86_avx_vtestc_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vtestps %ymm1, %ymm0 ; CHECK-NEXT: sbbl %eax, %eax ; CHECK-NEXT: andl $1, %eax @@ -3143,7 +3086,7 @@ declare i32 @llvm.x86.avx.vtestc.ps.256(<8 x float>, <8 x float>) nounwind readn define i32 @test_x86_avx_vtestnzc_pd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_avx_vtestnzc_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vtestpd %xmm1, %xmm0 ; CHECK-NEXT: seta %al ; CHECK-NEXT: movzbl %al, %eax @@ -3156,7 +3099,7 @@ declare i32 @llvm.x86.avx.vtestnzc.pd(<2 x double>, <2 x double>) nounwind readn define i32 @test_x86_avx_vtestnzc_pd_256(<4 x double> %a0, <4 x double> %a1) { ; CHECK-LABEL: test_x86_avx_vtestnzc_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vtestpd %ymm1, %ymm0 ; CHECK-NEXT: seta %al ; CHECK-NEXT: movzbl %al, %eax @@ -3170,7 +3113,7 @@ declare i32 @llvm.x86.avx.vtestnzc.pd.256(<4 x double>, <4 x double>) nounwind r define i32 @test_x86_avx_vtestnzc_ps(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_avx_vtestnzc_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vtestps %xmm1, %xmm0 ; CHECK-NEXT: seta %al ; CHECK-NEXT: movzbl %al, %eax @@ -3183,7 +3126,7 @@ declare i32 @llvm.x86.avx.vtestnzc.ps(<4 x float>, <4 x float>) nounwind readnon define i32 @test_x86_avx_vtestnzc_ps_256(<8 x float> %a0, <8 x float> %a1) { ; CHECK-LABEL: test_x86_avx_vtestnzc_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vtestps %ymm1, %ymm0 ; CHECK-NEXT: seta %al ; CHECK-NEXT: movzbl %al, %eax @@ -3197,7 +3140,7 @@ declare i32 @llvm.x86.avx.vtestnzc.ps.256(<8 x float>, <8 x float>) nounwind rea define i32 @test_x86_avx_vtestz_pd(<2 x double> %a0, <2 x double> %a1) { ; CHECK-LABEL: test_x86_avx_vtestz_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vtestpd %xmm1, %xmm0 ; CHECK-NEXT: sete %al ; CHECK-NEXT: movzbl %al, %eax @@ -3210,7 +3153,7 @@ declare i32 @llvm.x86.avx.vtestz.pd(<2 x double>, <2 x double>) nounwind readnon define i32 @test_x86_avx_vtestz_pd_256(<4 x double> %a0, <4 x double> %a1) { ; CHECK-LABEL: test_x86_avx_vtestz_pd_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vtestpd %ymm1, %ymm0 ; CHECK-NEXT: sete %al ; CHECK-NEXT: movzbl %al, %eax @@ -3224,7 +3167,7 @@ declare i32 @llvm.x86.avx.vtestz.pd.256(<4 x double>, <4 x double>) nounwind rea define i32 @test_x86_avx_vtestz_ps(<4 x float> %a0, <4 x float> %a1) { ; CHECK-LABEL: test_x86_avx_vtestz_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vtestps %xmm1, %xmm0 ; CHECK-NEXT: sete %al ; CHECK-NEXT: movzbl %al, %eax @@ -3237,7 +3180,7 @@ declare i32 @llvm.x86.avx.vtestz.ps(<4 x float>, <4 x float>) nounwind readnone define i32 @test_x86_avx_vtestz_ps_256(<8 x float> %a0, <8 x float> %a1) { ; CHECK-LABEL: test_x86_avx_vtestz_ps_256: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vtestps %ymm1, %ymm0 ; CHECK-NEXT: sete %al ; CHECK-NEXT: movzbl %al, %eax @@ -3251,7 +3194,7 @@ declare i32 @llvm.x86.avx.vtestz.ps.256(<8 x float>, <8 x float>) nounwind readn define void @test_x86_avx_vzeroall() { ; CHECK-LABEL: test_x86_avx_vzeroall: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vzeroall ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retl @@ -3263,7 +3206,7 @@ declare void @llvm.x86.avx.vzeroall() nounwind define void @test_x86_avx_vzeroupper() { ; CHECK-LABEL: test_x86_avx_vzeroupper: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retl @@ -3276,7 +3219,7 @@ declare void @llvm.x86.avx.vzeroupper() nounwind define void @monitor(i8* %P, i32 %E, i32 %H) nounwind { ; CHECK-LABEL: monitor: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax @@ -3290,7 +3233,7 @@ declare void @llvm.x86.sse3.monitor(i8*, i32, i32) nounwind define void @mwait(i32 %E, i32 %H) nounwind { ; CHECK-LABEL: mwait: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: mwait @@ -3302,7 +3245,7 @@ declare void @llvm.x86.sse3.mwait(i32, i32) nounwind define void @sfence() nounwind { ; CHECK-LABEL: sfence: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: sfence ; CHECK-NEXT: retl tail call void @llvm.x86.sse.sfence() @@ -3312,7 +3255,7 @@ declare void @llvm.x86.sse.sfence() nounwind define void @lfence() nounwind { ; CHECK-LABEL: lfence: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: lfence ; CHECK-NEXT: retl tail call void @llvm.x86.sse2.lfence() @@ -3322,7 +3265,7 @@ declare void @llvm.x86.sse2.lfence() nounwind define void @mfence() nounwind { ; CHECK-LABEL: mfence: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: mfence ; CHECK-NEXT: retl tail call void @llvm.x86.sse2.mfence() @@ -3332,7 +3275,7 @@ declare void @llvm.x86.sse2.mfence() nounwind define void @clflush(i8* %p) nounwind { ; CHECK-LABEL: clflush: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: clflush (%eax) ; CHECK-NEXT: retl @@ -3343,7 +3286,7 @@ declare void @llvm.x86.sse2.clflush(i8*) nounwind define i32 @crc32_32_8(i32 %a, i8 %b) nounwind { ; CHECK-LABEL: crc32_32_8: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: crc32b {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: retl @@ -3354,7 +3297,7 @@ declare i32 @llvm.x86.sse42.crc32.32.8(i32, i8) nounwind define i32 @crc32_32_16(i32 %a, i16 %b) nounwind { ; CHECK-LABEL: crc32_32_16: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: crc32w {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: retl @@ -3365,7 +3308,7 @@ declare i32 @llvm.x86.sse42.crc32.32.16(i32, i16) nounwind define i32 @crc32_32_32(i32 %a, i32 %b) nounwind { ; CHECK-LABEL: crc32_32_32: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: crc32l {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: retl @@ -3376,9 +3319,9 @@ declare i32 @llvm.x86.sse42.crc32.32.32(i32, i32) nounwind define void @movnt_dq(i8* %p, <2 x i64> %a1) nounwind { ; CHECK-LABEL: movnt_dq: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax -; CHECK-NEXT: vpaddq LCPI282_0, %xmm0, %xmm0 +; CHECK-NEXT: vpaddq LCPI277_0, %xmm0, %xmm0 ; CHECK-NEXT: vmovntdq %ymm0, (%eax) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retl @@ -3391,7 +3334,7 @@ declare void @llvm.x86.avx.movnt.dq.256(i8*, <4 x i64>) nounwind define void @movnt_ps(i8* %p, <8 x float> %a) nounwind { ; CHECK-LABEL: movnt_ps: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vmovntps %ymm0, (%eax) ; CHECK-NEXT: vzeroupper @@ -3404,7 +3347,7 @@ declare void @llvm.x86.avx.movnt.ps.256(i8*, <8 x float>) nounwind define void @movnt_pd(i8* %p, <4 x double> %a1) nounwind { ; add operation forces the execution domain. ; CHECK-LABEL: movnt_pd: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: vxorpd %ymm1, %ymm1, %ymm1 ; CHECK-NEXT: vaddpd %ymm1, %ymm0, %ymm0 @@ -3421,7 +3364,7 @@ declare void @llvm.x86.avx.movnt.pd.256(i8*, <4 x double>) nounwind ; Check for pclmulqdq define <2 x i64> @test_x86_pclmulqdq(<2 x i64> %a0, <2 x i64> %a1) { ; CHECK-LABEL: test_x86_pclmulqdq: -; CHECK: # BB#0: +; CHECK: ## BB#0: ; CHECK-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retl %res = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %a0, <2 x i64> %a1, i8 0) ; <<2 x i64>> [#uses=1] diff --git a/test/CodeGen/X86/avx-isa-check.ll b/test/CodeGen/X86/avx-isa-check.ll new file mode 100644 index 000000000000..77bfbd4bb423 --- /dev/null +++ b/test/CodeGen/X86/avx-isa-check.ll @@ -0,0 +1,570 @@ +; check AVX2 instructions that are disabled in case avx512VL/avx512BW present + +; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mcpu=core-avx2 -mattr=+avx2 -o /dev/null +; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mcpu=knl -o /dev/null +; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mcpu=knl -mattr=+avx512vl -o /dev/null +; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mcpu=knl -mattr=+avx512bw -o /dev/null +; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mcpu=knl -mattr=+avx512vl -mattr=+avx512bw -o /dev/null +; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mcpu=skx -o /dev/null + +define <4 x i64> @vpand_256(<4 x i64> %a, <4 x i64> %b) nounwind uwtable readnone ssp { + ; Force the execution domain with an add. + %a2 = add <4 x i64> %a, + %x = and <4 x i64> %a2, %b + ret <4 x i64> %x +} + +define <2 x i64> @vpand_128(<2 x i64> %a, <2 x i64> %b) nounwind uwtable readnone ssp { + ; Force the execution domain with an add. + %a2 = add <2 x i64> %a, + %x = and <2 x i64> %a2, %b + ret <2 x i64> %x +} + +define <4 x i64> @vpandn_256(<4 x i64> %a, <4 x i64> %b) nounwind uwtable readnone ssp { + ; Force the execution domain with an add. + %a2 = add <4 x i64> %a, + %y = xor <4 x i64> %a2, + %x = and <4 x i64> %a, %y + ret <4 x i64> %x +} + +define <2 x i64> @vpandn_128(<2 x i64> %a, <2 x i64> %b) nounwind uwtable readnone ssp { + ; Force the execution domain with an add. + %a2 = add <2 x i64> %a, + %y = xor <2 x i64> %a2, + %x = and <2 x i64> %a, %y + ret <2 x i64> %x +} + +define <4 x i64> @vpor_256(<4 x i64> %a, <4 x i64> %b) nounwind uwtable readnone ssp { + ; Force the execution domain with an add. + %a2 = add <4 x i64> %a, + %x = or <4 x i64> %a2, %b + ret <4 x i64> %x +} + +define <4 x i64> @vpxor_256(<4 x i64> %a, <4 x i64> %b) nounwind uwtable readnone ssp { + ; Force the execution domain with an add. + %a2 = add <4 x i64> %a, + %x = xor <4 x i64> %a2, %b + ret <4 x i64> %x +} + +define <2 x i64> @vpor_128(<2 x i64> %a, <2 x i64> %b) nounwind uwtable readnone ssp { + ; Force the execution domain with an add. + %a2 = add <2 x i64> %a, + %x = or <2 x i64> %a2, %b + ret <2 x i64> %x +} + +define <2 x i64> @vpxor_128(<2 x i64> %a, <2 x i64> %b) nounwind uwtable readnone ssp { + ; Force the execution domain with an add. + %a2 = add <2 x i64> %a, + %x = xor <2 x i64> %a2, %b + ret <2 x i64> %x +} + +define <4 x i64> @test_vpaddq_256(<4 x i64> %i, <4 x i64> %j) nounwind readnone { + %x = add <4 x i64> %i, %j + ret <4 x i64> %x +} + +define <8 x i32> @test_vpaddd_256(<8 x i32> %i, <8 x i32> %j) nounwind readnone { + %x = add <8 x i32> %i, %j + ret <8 x i32> %x +} + +define <16 x i16> @test_vpaddw_256(<16 x i16> %i, <16 x i16> %j) nounwind readnone { + %x = add <16 x i16> %i, %j + ret <16 x i16> %x +} + +define <32 x i8> @test_vpaddb_256(<32 x i8> %i, <32 x i8> %j) nounwind readnone { + %x = add <32 x i8> %i, %j + ret <32 x i8> %x +} + +define <4 x i64> @test_vpsubq_256(<4 x i64> %i, <4 x i64> %j) nounwind readnone { + %x = sub <4 x i64> %i, %j + ret <4 x i64> %x +} + +define <8 x i32> @test_vpsubd_256(<8 x i32> %i, <8 x i32> %j) nounwind readnone { + %x = sub <8 x i32> %i, %j + ret <8 x i32> %x +} + +define <16 x i16> @test_vpsubw_256(<16 x i16> %i, <16 x i16> %j) nounwind readnone { + %x = sub <16 x i16> %i, %j + ret <16 x i16> %x +} + +define <32 x i8> @test_vpsubb_256(<32 x i8> %i, <32 x i8> %j) nounwind readnone { + %x = sub <32 x i8> %i, %j + ret <32 x i8> %x +} + +define <16 x i16> @test_vpmullw_256(<16 x i16> %i, <16 x i16> %j) nounwind readnone { + %x = mul <16 x i16> %i, %j + ret <16 x i16> %x +} + +define <8 x i32> @test_vpcmpgtd_256(<8 x i32> %i, <8 x i32> %j) nounwind readnone { + %bincmp = icmp slt <8 x i32> %i, %j + %x = sext <8 x i1> %bincmp to <8 x i32> + ret <8 x i32> %x +} + +define <32 x i8> @test_vpcmpeqb_256(<32 x i8> %i, <32 x i8> %j) nounwind readnone { + %bincmp = icmp eq <32 x i8> %i, %j + %x = sext <32 x i1> %bincmp to <32 x i8> + ret <32 x i8> %x +} + +define <16 x i16> @test_vpcmpeqw_256(<16 x i16> %i, <16 x i16> %j) nounwind readnone { + %bincmp = icmp eq <16 x i16> %i, %j + %x = sext <16 x i1> %bincmp to <16 x i16> + ret <16 x i16> %x +} + +define <32 x i8> @test_vpcmpgtb_256(<32 x i8> %i, <32 x i8> %j) nounwind readnone { + %bincmp = icmp slt <32 x i8> %i, %j + %x = sext <32 x i1> %bincmp to <32 x i8> + ret <32 x i8> %x +} + +define <16 x i16> @test_vpcmpgtw_256(<16 x i16> %i, <16 x i16> %j) nounwind readnone { + %bincmp = icmp slt <16 x i16> %i, %j + %x = sext <16 x i1> %bincmp to <16 x i16> + ret <16 x i16> %x +} + +define <8 x i32> @test_vpcmpeqd_256(<8 x i32> %i, <8 x i32> %j) nounwind readnone { + %bincmp = icmp eq <8 x i32> %i, %j + %x = sext <8 x i1> %bincmp to <8 x i32> + ret <8 x i32> %x +} + +define <2 x i64> @test_vpaddq_128(<2 x i64> %i, <2 x i64> %j) nounwind readnone { + %x = add <2 x i64> %i, %j + ret <2 x i64> %x +} + +define <4 x i32> @test_vpaddd_128(<4 x i32> %i, <4 x i32> %j) nounwind readnone { + %x = add <4 x i32> %i, %j + ret <4 x i32> %x +} + +define <8 x i16> @test_vpaddw_128(<8 x i16> %i, <8 x i16> %j) nounwind readnone { + %x = add <8 x i16> %i, %j + ret <8 x i16> %x +} + +define <16 x i8> @test_vpaddb_128(<16 x i8> %i, <16 x i8> %j) nounwind readnone { + %x = add <16 x i8> %i, %j + ret <16 x i8> %x +} + +define <2 x i64> @test_vpsubq_128(<2 x i64> %i, <2 x i64> %j) nounwind readnone { + %x = sub <2 x i64> %i, %j + ret <2 x i64> %x +} + +define <4 x i32> @test_vpsubd_128(<4 x i32> %i, <4 x i32> %j) nounwind readnone { + %x = sub <4 x i32> %i, %j + ret <4 x i32> %x +} + +define <8 x i16> @test_vpsubw_128(<8 x i16> %i, <8 x i16> %j) nounwind readnone { + %x = sub <8 x i16> %i, %j + ret <8 x i16> %x +} + +define <16 x i8> @test_vpsubb_128(<16 x i8> %i, <16 x i8> %j) nounwind readnone { + %x = sub <16 x i8> %i, %j + ret <16 x i8> %x +} + +define <8 x i16> @test_vpmullw_128(<8 x i16> %i, <8 x i16> %j) nounwind readnone { + %x = mul <8 x i16> %i, %j + ret <8 x i16> %x +} + +define <8 x i16> @test_vpcmpgtw_128(<8 x i16> %i, <8 x i16> %j) nounwind readnone { + %bincmp = icmp slt <8 x i16> %i, %j + %x = sext <8 x i1> %bincmp to <8 x i16> + ret <8 x i16> %x +} + +define <16 x i8> @test_vpcmpgtb_128(<16 x i8> %i, <16 x i8> %j) nounwind readnone { + %bincmp = icmp slt <16 x i8> %i, %j + %x = sext <16 x i1> %bincmp to <16 x i8> + ret <16 x i8> %x +} + +define <8 x i16> @test_vpcmpeqw_128(<8 x i16> %i, <8 x i16> %j) nounwind readnone { + %bincmp = icmp eq <8 x i16> %i, %j + %x = sext <8 x i1> %bincmp to <8 x i16> + ret <8 x i16> %x +} + +define <16 x i8> @test_vpcmpeqb_128(<16 x i8> %i, <16 x i8> %j) nounwind readnone { + %bincmp = icmp eq <16 x i8> %i, %j + %x = sext <16 x i1> %bincmp to <16 x i8> + ret <16 x i8> %x +} + +define <8 x i16> @shuffle_v8i16_vpalignr(<8 x i16> %a, <8 x i16> %b) { + %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> + ret <8 x i16> %shuffle +} + +define <16 x i16> @shuffle_v16i16_vpalignr(<16 x i16> %a, <16 x i16> %b) { + %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> + ret <16 x i16> %shuffle +} + +define <16 x i8> @shuffle_v16i8_vpalignr(<16 x i8> %a, <16 x i8> %b) { + %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> + ret <16 x i8> %shuffle +} + +define <32 x i8> @shuffle_v32i8_vpalignr(<32 x i8> %a, <32 x i8> %b) { + %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> + ret <32 x i8> %shuffle +} + +define <2 x i64> @shuffle_v2i64_vpalignr(<2 x i64> %a, <2 x i64> %b) { + %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> + ret <2 x i64> %shuffle +} + +define <4 x i32> @shuffle_v4i32_vpalignr(<4 x i32> %a, <4 x i32> %b) { + %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> + ret <4 x i32> %shuffle +} + +define <8 x i32> @shuffle_v8i32_vpalignr(<8 x i32> %a, <8 x i32> %b) { + %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> + ret <8 x i32> %shuffle +} + +define <4 x double> @shuffle_v4f64_5163(<4 x double> %a, <4 x double> %b) { + %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> + ret <4 x double> %shuffle +} + +define <2 x double> @shuffle_v2f64_bitcast_1z(<2 x double> %a) { + %shuffle64 = shufflevector <2 x double> %a, <2 x double> zeroinitializer, <2 x i32> + %bitcast32 = bitcast <2 x double> %shuffle64 to <4 x float> + %shuffle32 = shufflevector <4 x float> %bitcast32, <4 x float> undef, <4 x i32> + %bitcast64 = bitcast <4 x float> %shuffle32 to <2 x double> + ret <2 x double> %bitcast64 +} + +define <16 x i16> @shuffle_v16i16_zz_zz_zz_zz_zz_zz_zz_16_zz_zz_zz_zz_zz_zz_zz_24(<16 x i16> %a) { + %shuffle = shufflevector <16 x i16> zeroinitializer, <16 x i16> %a, <16 x i32> + ret <16 x i16> %shuffle +} + +define i64 @extract_v2i64(<2 x i64> %x, i64* %dst) { + %r1 = extractelement <2 x i64> %x, i32 0 + %r2 = extractelement <2 x i64> %x, i32 1 + store i64 %r2, i64* %dst, align 1 + ret i64 %r1 +} + +define i32 @extract_v4i32(<4 x i32> %x, i32* %dst) { + %r1 = extractelement <4 x i32> %x, i32 1 + %r2 = extractelement <4 x i32> %x, i32 3 + store i32 %r2, i32* %dst, align 1 + ret i32 %r1 +} + +define i16 @extract_v8i16(<8 x i16> %x, i16* %dst) { + %r1 = extractelement <8 x i16> %x, i32 1 + %r2 = extractelement <8 x i16> %x, i32 3 + store i16 %r2, i16* %dst, align 1 + ret i16 %r1 +} + +define i8 @extract_v16i8(<16 x i8> %x, i8* %dst) { + %r1 = extractelement <16 x i8> %x, i32 1 + %r2 = extractelement <16 x i8> %x, i32 3 + store i8 %r2, i8* %dst, align 1 + ret i8 %r1 +} + +define <2 x i64> @insert_v2i64(<2 x i64> %x, i64 %y , i64* %ptr) { + %val = load i64, i64* %ptr + %r1 = insertelement <2 x i64> %x, i64 %val, i32 1 + %r2 = insertelement <2 x i64> %r1, i64 %y, i32 3 + ret <2 x i64> %r2 +} + +define <4 x i32> @insert_v4i32(<4 x i32> %x, i32 %y, i32* %ptr) { + %val = load i32, i32* %ptr + %r1 = insertelement <4 x i32> %x, i32 %val, i32 1 + %r2 = insertelement <4 x i32> %r1, i32 %y, i32 3 + ret <4 x i32> %r2 +} + +define <8 x i16> @insert_v8i16(<8 x i16> %x, i16 %y, i16* %ptr) { + %val = load i16, i16* %ptr + %r1 = insertelement <8 x i16> %x, i16 %val, i32 1 + %r2 = insertelement <8 x i16> %r1, i16 %y, i32 5 + ret <8 x i16> %r2 +} + +define <16 x i8> @insert_v16i8(<16 x i8> %x, i8 %y, i8* %ptr) { + %val = load i8, i8* %ptr + %r1 = insertelement <16 x i8> %x, i8 %val, i32 3 + %r2 = insertelement <16 x i8> %r1, i8 %y, i32 10 + ret <16 x i8> %r2 +} + +define <4 x i32> @shuffle_v4i32_0451(<4 x i32> %a, <4 x i32> %b) { + %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> + ret <4 x i32> %shuffle +} + +define <4 x i32> @shuffle_v4i32_0142(<4 x i32> %a, <4 x i32> %b) { + %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> + ret <4 x i32> %shuffle +} + +define <16 x i8> @shuffle_v16i8_0101010101010101(<16 x i8> %a, <16 x i8> %b) { + %shuffle = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> + ret <16 x i8> %shuffle +} + +define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00(<16 x i16> %a, <16 x i16> %b) { + %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> + ret <16 x i16> %shuffle +} + +define <8 x float> @shuffle_v8f32_11335577(<8 x float> %a, <8 x float> %b) { +; vmovshdup 256 test + %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> + ret <8 x float> %shuffle +} + +define <4 x float> @shuffle_v4f32_1133(<4 x float> %a, <4 x float> %b) { +; vmovshdup 128 test + %shuffle = shufflevector <4 x float> %a, <4 x float> %b, <4 x i32> + ret <4 x float> %shuffle +} + +define <8 x float> @shuffle_v8f32_00224466(<8 x float> %a, <8 x float> %b) { +; vmovsldup 256 test + %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> + ret <8 x float> %shuffle +} + +define <4 x float> @shuffle_v4f32_0022(<4 x float> %a, <4 x float> %b) { +; vmovsldup 128 test + %shuffle = shufflevector <4 x float> %a, <4 x float> %b, <4 x i32> + ret <4 x float> %shuffle +} + +define <2 x double> @insert_mem_lo_v2f64(double* %ptr, <2 x double> %b) { + %a = load double, double* %ptr + %v = insertelement <2 x double> undef, double %a, i32 0 + %shuffle = shufflevector <2 x double> %v, <2 x double> %b, <2 x i32> + ret <2 x double> %shuffle +} + +define <2 x double> @insert_mem_hi_v2f64(double* %ptr, <2 x double> %b) { + %a = load double, double* %ptr + %v = insertelement <2 x double> undef, double %a, i32 0 + %shuffle = shufflevector <2 x double> %v, <2 x double> %b, <2 x i32> + ret <2 x double> %shuffle +} + +define void @store_floats(<4 x float> %x, i64* %p) { + %a = fadd <4 x float> %x, %x + %b = shufflevector <4 x float> %a, <4 x float> undef, <2 x i32> + %c = bitcast <2 x float> %b to i64 + store i64 %c, i64* %p + ret void +} + +define void @store_double(<2 x double> %x, i64* %p) { + %a = fadd <2 x double> %x, %x + %b = extractelement <2 x double> %a, i32 0 + %c = bitcast double %b to i64 + store i64 %c, i64* %p + ret void +} + +define void @store_h_double(<2 x double> %x, i64* %p) { + %a = fadd <2 x double> %x, %x + %b = extractelement <2 x double> %a, i32 1 + %c = bitcast double %b to i64 + store i64 %c, i64* %p + ret void +} + +define <2 x double> @test39(double* %ptr) nounwind { + %a = load double, double* %ptr + %v = insertelement <2 x double> undef, double %a, i32 0 + %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> + ret <2 x double> %shuffle + } + +define <2 x double> @test40(<2 x double>* %ptr) nounwind { + %v = load <2 x double>, <2 x double>* %ptr + %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> + ret <2 x double> %shuffle + } + +define <2 x double> @shuffle_v2f64_00(<2 x double> %a, <2 x double> %b) { + %shuffle = shufflevector <2 x double> %a, <2 x double> %b, <2 x i32> + ret <2 x double> %shuffle +} + +define <4 x double> @shuffle_v4f64_0022(<4 x double> %a, <4 x double> %b) { + %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> + ret <4 x double> %shuffle +} + +define <8 x i32> @ashr_v8i32(<8 x i32> %a, <8 x i32> %b) { + %shift = ashr <8 x i32> %a, %b + ret <8 x i32> %shift +} + +define <8 x i32> @lshr_v8i32(<8 x i32> %a, <8 x i32> %b) { + %shift = lshr <8 x i32> %a, %b + ret <8 x i32> %shift +} + +define <8 x i32> @shl_v8i32(<8 x i32> %a, <8 x i32> %b) { + %shift = shl <8 x i32> %a, %b + ret <8 x i32> %shift +} + +define <8 x i32> @ashr_const_v8i32(<8 x i32> %a) { + %shift = ashr <8 x i32> %a, + ret <8 x i32> %shift +} + +define <8 x i32> @lshr_const_v8i32(<8 x i32> %a) { + %shift = lshr <8 x i32> %a, + ret <8 x i32> %shift +} + +define <8 x i32> @shl_const_v8i32(<8 x i32> %a) { + %shift = shl <8 x i32> %a, + ret <8 x i32> %shift +} + +define <4 x i64> @ashr_v4i64(<4 x i64> %a, <4 x i64> %b) { + %shift = ashr <4 x i64> %a, %b + ret <4 x i64> %shift +} + +define <4 x i64> @lshr_v4i64(<4 x i64> %a, <4 x i64> %b) { + %shift = lshr <4 x i64> %a, %b + ret <4 x i64> %shift +} + +define <4 x i64> @shl_v4i64(<4 x i64> %a, <4 x i64> %b) { + %shift = shl <4 x i64> %a, %b + ret <4 x i64> %shift +} + +define <4 x i64> @ashr_const_v4i64(<4 x i64> %a) { + %shift = ashr <4 x i64> %a, + ret <4 x i64> %shift +} + +define <4 x i64> @lshr_const_v4i64(<4 x i64> %a) { + %shift = lshr <4 x i64> %a, + ret <4 x i64> %shift +} + +define <4 x i64> @shl_const_v4i64(<4 x i64> %a) { + %shift = shl <4 x i64> %a, + ret <4 x i64> %shift +} + +define <16 x i16> @ashr_v16i16(<16 x i16> %a, <16 x i16> %b) { + %shift = ashr <16 x i16> %a, %b + ret <16 x i16> %shift +} + +define <16 x i16> @lshr_v16i16(<16 x i16> %a, <16 x i16> %b) { + %shift = lshr <16 x i16> %a, %b + ret <16 x i16> %shift +} + +define <16 x i16> @shl_v16i16(<16 x i16> %a, <16 x i16> %b) { + %shift = shl <16 x i16> %a, %b + ret <16 x i16> %shift +} + +define <16 x i16> @ashr_const_v16i16(<16 x i16> %a) { + %shift = ashr <16 x i16> %a, + ret <16 x i16> %shift +} + +define <16 x i16> @lshr_const_v16i16(<16 x i16> %a) { + %shift = lshr <16 x i16> %a, + ret <16 x i16> %shift +} + +define <16 x i16> @shl_const_v16i16(<16 x i16> %a) { + %shift = shl <16 x i16> %a, + ret <16 x i16> %shift +} + +define <4 x i32> @ashr_v4i32(<4 x i32> %a, <4 x i32> %b) { + %shift = ashr <4 x i32> %a, %b + ret <4 x i32> %shift +} + +define <4 x i32> @shl_const_v4i32(<4 x i32> %a) { + %shift = shl <4 x i32> %a, + ret <4 x i32> %shift +} + +define <2 x i64> @ashr_v2i64(<2 x i64> %a, <2 x i64> %b) { + %shift = ashr <2 x i64> %a, %b + ret <2 x i64> %shift +} + +define <2 x i64> @shl_const_v2i64(<2 x i64> %a) { + %shift = shl <2 x i64> %a, + ret <2 x i64> %shift +} + +define <8 x i16> @ashr_v8i16(<8 x i16> %a, <8 x i16> %b) { + %shift = ashr <8 x i16> %a, %b + ret <8 x i16> %shift +} + +define <8 x i16> @lshr_v8i16(<8 x i16> %a, <8 x i16> %b) { + %shift = lshr <8 x i16> %a, %b + ret <8 x i16> %shift +} + +define <8 x i16> @shl_v8i16(<8 x i16> %a, <8 x i16> %b) { + %shift = shl <8 x i16> %a, %b + ret <8 x i16> %shift +} + +define <8 x i16> @ashr_const_v8i16(<8 x i16> %a) { + %shift = ashr <8 x i16> %a, + ret <8 x i16> %shift +} + +define <8 x i16> @lshr_const_v8i16(<8 x i16> %a) { + %shift = lshr <8 x i16> %a, + ret <8 x i16> %shift +} + +define <8 x i16> @shl_const_v8i16(<8 x i16> %a) { + %shift = shl <8 x i16> %a, + ret <8 x i16> %shift +} diff --git a/test/CodeGen/X86/avx-load-store.ll b/test/CodeGen/X86/avx-load-store.ll index d2f213bac7bb..d7eceb7cce66 100644 --- a/test/CodeGen/X86/avx-load-store.ll +++ b/test/CodeGen/X86/avx-load-store.ll @@ -88,7 +88,7 @@ entry: ret void } -declare void @llvm.x86.avx.maskstore.ps.256(i8*, <8 x float>, <8 x float>) nounwind +declare void @llvm.x86.avx.maskstore.ps.256(i8*, <8 x i32>, <8 x float>) nounwind ; CHECK_O0: _f_f ; CHECK-O0: vmovss LCPI @@ -105,7 +105,7 @@ cif_mask_mixed: ; preds = %allocas br i1 undef, label %cif_mixed_test_all, label %cif_mixed_test_any_check cif_mixed_test_all: ; preds = %cif_mask_mixed - call void @llvm.x86.avx.maskstore.ps.256(i8* undef, <8 x float> , <8 x float> undef) nounwind + call void @llvm.x86.avx.maskstore.ps.256(i8* undef, <8 x i32> , <8 x float> undef) nounwind unreachable cif_mixed_test_any_check: ; preds = %cif_mask_mixed diff --git a/test/CodeGen/X86/avx-logic.ll b/test/CodeGen/X86/avx-logic.ll index e71ac473b44d..e9e7d5aea273 100644 --- a/test/CodeGen/X86/avx-logic.ll +++ b/test/CodeGen/X86/avx-logic.ll @@ -1,4 +1,6 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s define <4 x double> @andpd256(<4 x double> %y, <4 x double> %x) nounwind uwtable readnone ssp { ; CHECK-LABEL: andpd256: diff --git a/test/CodeGen/X86/avx-shift.ll b/test/CodeGen/X86/avx-shift.ll index 83585b536095..033a95276608 100644 --- a/test/CodeGen/X86/avx-shift.ll +++ b/test/CodeGen/X86/avx-shift.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s ;;; Shift left diff --git a/test/CodeGen/X86/avx-shuffle-x86_32.ll b/test/CodeGen/X86/avx-shuffle-x86_32.ll index 78b4888cfa16..fae5b41abfa6 100755 --- a/test/CodeGen/X86/avx-shuffle-x86_32.ll +++ b/test/CodeGen/X86/avx-shuffle-x86_32.ll @@ -1,8 +1,26 @@ -; RUN: llc < %s -mtriple=i686-pc-win32 -mcpu=corei7-avx -mattr=+avx | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx | FileCheck %s +; Avoid unnecessary vinsertf128 define <4 x i64> @test1(<4 x i64> %a) nounwind { +; CHECK-LABEL: test1: +; CHECK: # BB#0: +; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm1 +; CHECK-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] +; CHECK-NEXT: retl %b = shufflevector <4 x i64> %a, <4 x i64> undef, <4 x i32> ret <4 x i64>%b - ; CHECK-LABEL: test1: - ; CHECK-NOT: vinsertf128 - } +} + +define <8 x i16> @test2(<4 x i16>* %v) nounwind { +; CHECK-LABEL: test2: +; CHECK: # BB#0: +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax +; CHECK-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero +; CHECK-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; CHECK-NEXT: retl + %v9 = load <4 x i16>, <4 x i16> * %v, align 8 + %v10 = shufflevector <4 x i16> %v9, <4 x i16> undef, <8 x i32> + %v11 = shufflevector <8 x i16> , <8 x i16> %v10, <8 x i32> + ret <8 x i16> %v11 +} diff --git a/test/CodeGen/X86/avx-splat.ll b/test/CodeGen/X86/avx-splat.ll index 3ea7e386c426..ebaaf0e8d00d 100644 --- a/test/CodeGen/X86/avx-splat.ll +++ b/test/CodeGen/X86/avx-splat.ll @@ -1,26 +1,34 @@ -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7-avx -mattr=+avx | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx | FileCheck %s - -; CHECK: vpshufb {{.*}} ## xmm0 = xmm0[5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5] -; CHECK-NEXT: vinsertf128 $1 define <32 x i8> @funcA(<32 x i8> %a) nounwind uwtable readnone ssp { +; CHECK-LABEL: funcA: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5] +; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; CHECK-NEXT: retq entry: %shuffle = shufflevector <32 x i8> %a, <32 x i8> undef, <32 x i32> ret <32 x i8> %shuffle } -; CHECK: vpshufb {{.*}} ## xmm0 = xmm0[10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11] -; CHECK-NEXT: vinsertf128 $1 define <16 x i16> @funcB(<16 x i16> %a) nounwind uwtable readnone ssp { +; CHECK-LABEL: funcB: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11] +; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; CHECK-NEXT: retq entry: %shuffle = shufflevector <16 x i16> %a, <16 x i16> undef, <16 x i32> ret <16 x i16> %shuffle } -; CHECK: vmovq -; CHECK-NEXT: vmovddup %xmm -; CHECK-NEXT: vinsertf128 $1 define <4 x i64> @funcC(i64 %q) nounwind uwtable readnone ssp { +; CHECK-LABEL: funcC: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vmovq %rdi, %xmm0 +; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0] +; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; CHECK-NEXT: retq entry: %vecinit.i = insertelement <4 x i64> undef, i64 %q, i32 0 %vecinit2.i = insertelement <4 x i64> %vecinit.i, i64 %q, i32 1 @@ -29,9 +37,12 @@ entry: ret <4 x i64> %vecinit6.i } -; CHECK: vmovddup %xmm -; CHECK-NEXT: vinsertf128 $1 define <4 x double> @funcD(double %q) nounwind uwtable readnone ssp { +; CHECK-LABEL: funcD: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0] +; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; CHECK-NEXT: retq entry: %vecinit.i = insertelement <4 x double> undef, double %q, i32 0 %vecinit2.i = insertelement <4 x double> %vecinit.i, double %q, i32 1 @@ -43,8 +54,23 @@ entry: ; Test this turns into a broadcast: ; shuffle (scalar_to_vector (load (ptr + 4))), undef, <0, 0, 0, 0> ; -; CHECK: vbroadcastss define <8 x float> @funcE() nounwind { +; CHECK-LABEL: funcE: +; CHECK: ## BB#0: ## %for_exit499 +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: ## implicit-def: %YMM0 +; CHECK-NEXT: testb %al, %al +; CHECK-NEXT: jne LBB4_2 +; CHECK-NEXT: ## BB#1: ## %load.i1247 +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: movq %rsp, %rbp +; CHECK-NEXT: andq $-32, %rsp +; CHECK-NEXT: subq $1312, %rsp ## imm = 0x520 +; CHECK-NEXT: vbroadcastss {{[0-9]+}}(%rsp), %ymm0 +; CHECK-NEXT: movq %rbp, %rsp +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: LBB4_2: ## %__load_and_broadcast_32.exit1249 +; CHECK-NEXT: retq allocas: %udx495 = alloca [18 x [18 x float]], align 32 br label %for_test505.preheader @@ -69,29 +95,79 @@ __load_and_broadcast_32.exit1249: ; preds = %load.i1247, %for_ex ret <8 x float> %load_broadcast12281250 } -; CHECK: vpermilps $4 -; CHECK-NEXT: vinsertf128 $1 define <8 x float> @funcF(i32 %val) nounwind { +; CHECK-LABEL: funcF: +; CHECK: ## BB#0: +; CHECK-NEXT: vmovd %edi, %xmm0 +; CHECK-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,0] +; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; CHECK-NEXT: retq %ret6 = insertelement <8 x i32> undef, i32 %val, i32 6 %ret7 = insertelement <8 x i32> %ret6, i32 %val, i32 7 %tmp = bitcast <8 x i32> %ret7 to <8 x float> ret <8 x float> %tmp } -; CHECK: vpermilps $0 -; CHECK-NEXT: vinsertf128 $1 define <8 x float> @funcG(<8 x float> %a) nounwind uwtable readnone ssp { +; CHECK-LABEL: funcG: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[0,0,0,0] +; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; CHECK-NEXT: retq entry: %shuffle = shufflevector <8 x float> %a, <8 x float> undef, <8 x i32> ret <8 x float> %shuffle } -; CHECK: vextractf128 $1 -; CHECK-NEXT: vpermilps $85 -; CHECK-NEXT: vinsertf128 $1 define <8 x float> @funcH(<8 x float> %a) nounwind uwtable readnone ssp { +; CHECK-LABEL: funcH: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm0 +; CHECK-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[1,1,1,1] +; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; CHECK-NEXT: retq entry: %shuffle = shufflevector <8 x float> %a, <8 x float> undef, <8 x i32> ret <8 x float> %shuffle } +define <2 x double> @splat_load_2f64_11(<2 x double>* %ptr) { +; CHECK-LABEL: splat_load_2f64_11: +; CHECK: ## BB#0: +; CHECK-NEXT: vmovaps (%rdi), %xmm0 +; CHECK-NEXT: vmovhlps {{.*#+}} xmm0 = xmm0[1,1] +; CHECK-NEXT: retq + %x = load <2 x double>, <2 x double>* %ptr + %x1 = shufflevector <2 x double> %x, <2 x double> undef, <2 x i32> + ret <2 x double> %x1 +} + +define <4 x double> @splat_load_4f64_2222(<4 x double>* %ptr) { +; CHECK-LABEL: splat_load_4f64_2222: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastsd 16(%rdi), %ymm0 +; CHECK-NEXT: retq + %x = load <4 x double>, <4 x double>* %ptr + %x1 = shufflevector <4 x double> %x, <4 x double> undef, <4 x i32> + ret <4 x double> %x1 +} + +define <4 x float> @splat_load_4f32_0000(<4 x float>* %ptr) { +; CHECK-LABEL: splat_load_4f32_0000: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastss (%rdi), %xmm0 +; CHECK-NEXT: retq + %x = load <4 x float>, <4 x float>* %ptr + %x1 = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> + ret <4 x float> %x1 +} + +define <8 x float> @splat_load_8f32_77777777(<8 x float>* %ptr) { +; CHECK-LABEL: splat_load_8f32_77777777: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastss 28(%rdi), %ymm0 +; CHECK-NEXT: retq + %x = load <8 x float>, <8 x float>* %ptr + %x1 = shufflevector <8 x float> %x, <8 x float> undef, <8 x i32> + ret <8 x float> %x1 +} diff --git a/test/CodeGen/X86/avx-vbroadcast.ll b/test/CodeGen/X86/avx-vbroadcast.ll index 8b8c11b85875..86b0628aa0bc 100644 --- a/test/CodeGen/X86/avx-vbroadcast.ll +++ b/test/CodeGen/X86/avx-vbroadcast.ll @@ -1,7 +1,11 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7-avx -mattr=+avx | FileCheck %s -; CHECK: vbroadcastsd (% define <4 x i64> @A(i64* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: A: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastsd (%rdi), %ymm0 +; CHECK-NEXT: retq entry: %q = load i64, i64* %ptr, align 8 %vecinit.i = insertelement <4 x i64> undef, i64 %q, i32 0 @@ -11,8 +15,11 @@ entry: ret <4 x i64> %vecinit6.i } -; CHECK: vbroadcastss (% define <8 x i32> @B(i32* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: B: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss (%rdi), %ymm0 +; CHECK-NEXT: retq entry: %q = load i32, i32* %ptr, align 4 %vecinit.i = insertelement <8 x i32> undef, i32 %q, i32 0 @@ -22,8 +29,11 @@ entry: ret <8 x i32> %vecinit6.i } -; CHECK: vbroadcastsd (% define <4 x double> @C(double* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: C: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastsd (%rdi), %ymm0 +; CHECK-NEXT: retq entry: %q = load double, double* %ptr, align 8 %vecinit.i = insertelement <4 x double> undef, double %q, i32 0 @@ -33,8 +43,11 @@ entry: ret <4 x double> %vecinit6.i } -; CHECK: vbroadcastss (% define <8 x float> @D(float* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: D: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss (%rdi), %ymm0 +; CHECK-NEXT: retq entry: %q = load float, float* %ptr, align 4 %vecinit.i = insertelement <8 x float> undef, float %q, i32 0 @@ -46,8 +59,11 @@ entry: ;;;; 128-bit versions -; CHECK: vbroadcastss (% define <4 x float> @e(float* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: e: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss (%rdi), %xmm0 +; CHECK-NEXT: retq entry: %q = load float, float* %ptr, align 4 %vecinit.i = insertelement <4 x float> undef, float %q, i32 0 @@ -57,12 +73,14 @@ entry: ret <4 x float> %vecinit6.i } - -; CHECK: _e2 -; CHECK-NOT: vbroadcastss -; CHECK: ret +; Don't broadcast constants on pre-AVX2 hardware. define <4 x float> @_e2(float* %ptr) nounwind uwtable readnone ssp { - %vecinit.i = insertelement <4 x float> undef, float 0xbf80000000000000, i32 0 +; CHECK-LABEL: _e2: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vmovaps {{.*#+}} xmm0 = [-7.812500e-03,-7.812500e-03,-7.812500e-03,-7.812500e-03] +; CHECK-NEXT: retq +entry: + %vecinit.i = insertelement <4 x float> undef, float 0xbf80000000000000, i32 0 %vecinit2.i = insertelement <4 x float> %vecinit.i, float 0xbf80000000000000, i32 1 %vecinit4.i = insertelement <4 x float> %vecinit2.i, float 0xbf80000000000000, i32 2 %vecinit6.i = insertelement <4 x float> %vecinit4.i, float 0xbf80000000000000, i32 3 @@ -70,8 +88,11 @@ define <4 x float> @_e2(float* %ptr) nounwind uwtable readnone ssp { } -; CHECK: vbroadcastss (% define <4 x i32> @F(i32* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: F: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss (%rdi), %xmm0 +; CHECK-NEXT: retq entry: %q = load i32, i32* %ptr, align 4 %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0 @@ -81,12 +102,158 @@ entry: ret <4 x i32> %vecinit6.i } +; FIXME: Pointer adjusted broadcasts + +define <4 x i32> @load_splat_4i32_4i32_1111(<4 x i32>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_4i32_4i32_1111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpshufd {{.*#+}} xmm0 = mem[1,1,1,1] +; CHECK-NEXT: retq +entry: + %ld = load <4 x i32>, <4 x i32>* %ptr + %ret = shufflevector <4 x i32> %ld, <4 x i32> undef, <4 x i32> + ret <4 x i32> %ret +} + +define <8 x i32> @load_splat_8i32_4i32_33333333(<4 x i32>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_8i32_4i32_33333333: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpermilps {{.*#+}} xmm0 = mem[3,3,3,3] +; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <4 x i32>, <4 x i32>* %ptr + %ret = shufflevector <4 x i32> %ld, <4 x i32> undef, <8 x i32> + ret <8 x i32> %ret +} + +define <8 x i32> @load_splat_8i32_8i32_55555555(<8 x i32>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_8i32_8i32_55555555: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vmovaps (%rdi), %ymm0 +; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm0 +; CHECK-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[1,1,1,1] +; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <8 x i32>, <8 x i32>* %ptr + %ret = shufflevector <8 x i32> %ld, <8 x i32> undef, <8 x i32> + ret <8 x i32> %ret +} + +define <4 x float> @load_splat_4f32_4f32_1111(<4 x float>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_4f32_4f32_1111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss 4(%rdi), %xmm0 +; CHECK-NEXT: retq +entry: + %ld = load <4 x float>, <4 x float>* %ptr + %ret = shufflevector <4 x float> %ld, <4 x float> undef, <4 x i32> + ret <4 x float> %ret +} + +define <8 x float> @load_splat_8f32_4f32_33333333(<4 x float>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_8f32_4f32_33333333: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss 12(%rdi), %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <4 x float>, <4 x float>* %ptr + %ret = shufflevector <4 x float> %ld, <4 x float> undef, <8 x i32> + ret <8 x float> %ret +} + +define <8 x float> @load_splat_8f32_8f32_55555555(<8 x float>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_8f32_8f32_55555555: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss 20(%rdi), %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <8 x float>, <8 x float>* %ptr + %ret = shufflevector <8 x float> %ld, <8 x float> undef, <8 x i32> + ret <8 x float> %ret +} + +define <2 x i64> @load_splat_2i64_2i64_1111(<2 x i64>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_2i64_2i64_1111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpshufd {{.*#+}} xmm0 = mem[2,3,2,3] +; CHECK-NEXT: retq +entry: + %ld = load <2 x i64>, <2 x i64>* %ptr + %ret = shufflevector <2 x i64> %ld, <2 x i64> undef, <2 x i32> + ret <2 x i64> %ret +} + +define <4 x i64> @load_splat_4i64_2i64_1111(<2 x i64>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_4i64_2i64_1111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vmovaps (%rdi), %xmm0 +; CHECK-NEXT: vmovhlps {{.*#+}} xmm0 = xmm0[1,1] +; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <2 x i64>, <2 x i64>* %ptr + %ret = shufflevector <2 x i64> %ld, <2 x i64> undef, <4 x i32> + ret <4 x i64> %ret +} + +define <4 x i64> @load_splat_4i64_4i64_2222(<4 x i64>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_4i64_4i64_2222: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vmovapd (%rdi), %ymm0 +; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm0 +; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0] +; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <4 x i64>, <4 x i64>* %ptr + %ret = shufflevector <4 x i64> %ld, <4 x i64> undef, <4 x i32> + ret <4 x i64> %ret +} + +define <2 x double> @load_splat_2f64_2f64_1111(<2 x double>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_2f64_2f64_1111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vmovaps (%rdi), %xmm0 +; CHECK-NEXT: vmovhlps {{.*#+}} xmm0 = xmm0[1,1] +; CHECK-NEXT: retq +entry: + %ld = load <2 x double>, <2 x double>* %ptr + %ret = shufflevector <2 x double> %ld, <2 x double> undef, <2 x i32> + ret <2 x double> %ret +} + +define <4 x double> @load_splat_4f64_2f64_1111(<2 x double>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_4f64_2f64_1111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastsd 8(%rdi), %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <2 x double>, <2 x double>* %ptr + %ret = shufflevector <2 x double> %ld, <2 x double> undef, <4 x i32> + ret <4 x double> %ret +} + +define <4 x double> @load_splat_4f64_4f64_2222(<4 x double>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_4f64_4f64_2222: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastsd 16(%rdi), %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <4 x double>, <4 x double>* %ptr + %ret = shufflevector <4 x double> %ld, <4 x double> undef, <4 x i32> + ret <4 x double> %ret +} + ; Unsupported vbroadcasts -; CHECK: _G -; CHECK-NOT: broadcast (% -; CHECK: ret define <2 x i64> @G(i64* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: G: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; CHECK-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; CHECK-NEXT: retq entry: %q = load i64, i64* %ptr, align 8 %vecinit.i = insertelement <2 x i64> undef, i64 %q, i32 0 @@ -94,18 +261,21 @@ entry: ret <2 x i64> %vecinit2.i } -; CHECK: _H -; CHECK-NOT: broadcast -; CHECK: ret define <4 x i32> @H(<4 x i32> %a) { +; CHECK-LABEL: H: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; CHECK-NEXT: retq +entry: %x = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> ret <4 x i32> %x } -; CHECK: _I -; CHECK-NOT: broadcast (% -; CHECK: ret define <2 x double> @I(double* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: I: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = mem[0,0] +; CHECK-NEXT: retq entry: %q = load double, double* %ptr, align 4 %vecinit.i = insertelement <2 x double> undef, double %q, i32 0 @@ -113,10 +283,13 @@ entry: ret <2 x double> %vecinit2.i } -; CHECK: _RR -; CHECK: vbroadcastss (% -; CHECK: ret define <4 x float> @_RR(float* %ptr, i32* %k) nounwind uwtable readnone ssp { +; CHECK-LABEL: _RR: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss (%rdi), %xmm0 +; CHECK-NEXT: movl (%rsi), %eax +; CHECK-NEXT: movl %eax, (%rax) +; CHECK-NEXT: retq entry: %q = load float, float* %ptr, align 4 %vecinit.i = insertelement <4 x float> undef, float %q, i32 0 @@ -129,11 +302,11 @@ entry: ret <4 x float> %vecinit6.i } - -; CHECK: _RR2 -; CHECK: vbroadcastss (% -; CHECK: ret define <4 x float> @_RR2(float* %ptr, i32* %k) nounwind uwtable readnone ssp { +; CHECK-LABEL: _RR2: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss (%rdi), %xmm0 +; CHECK-NEXT: retq entry: %q = load float, float* %ptr, align 4 %v = insertelement <4 x float> undef, float %q, i32 0 @@ -141,16 +314,15 @@ entry: ret <4 x float> %t } - ; These tests check that a vbroadcast instruction is used when we have a splat ; formed from a concat_vectors (via the shufflevector) of two BUILD_VECTORs ; (via the insertelements). -; CHECK-LABEL: splat_concat1 -; CHECK-NOT: vinsertf128 -; CHECK: vbroadcastss (% -; CHECK-NEXT: ret define <8 x float> @splat_concat1(float* %p) { +; CHECK-LABEL: splat_concat1: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastss (%rdi), %ymm0 +; CHECK-NEXT: retq %1 = load float, float* %p, align 4 %2 = insertelement <4 x float> undef, float %1, i32 0 %3 = insertelement <4 x float> %2, float %1, i32 1 @@ -160,11 +332,11 @@ define <8 x float> @splat_concat1(float* %p) { ret <8 x float> %6 } -; CHECK-LABEL: splat_concat2 -; CHECK-NOT: vinsertf128 -; CHECK: vbroadcastss (% -; CHECK-NEXT: ret define <8 x float> @splat_concat2(float* %p) { +; CHECK-LABEL: splat_concat2: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastss (%rdi), %ymm0 +; CHECK-NEXT: retq %1 = load float, float* %p, align 4 %2 = insertelement <4 x float> undef, float %1, i32 0 %3 = insertelement <4 x float> %2, float %1, i32 1 @@ -178,11 +350,11 @@ define <8 x float> @splat_concat2(float* %p) { ret <8 x float> %10 } -; CHECK-LABEL: splat_concat3 -; CHECK-NOT: vinsertf128 -; CHECK: vbroadcastsd (% -; CHECK-NEXT: ret define <4 x double> @splat_concat3(double* %p) { +; CHECK-LABEL: splat_concat3: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastsd (%rdi), %ymm0 +; CHECK-NEXT: retq %1 = load double, double* %p, align 8 %2 = insertelement <2 x double> undef, double %1, i32 0 %3 = insertelement <2 x double> %2, double %1, i32 1 @@ -190,11 +362,11 @@ define <4 x double> @splat_concat3(double* %p) { ret <4 x double> %4 } -; CHECK-LABEL: splat_concat4 -; CHECK-NOT: vinsertf128 -; CHECK: vbroadcastsd (% -; CHECK-NEXT: ret define <4 x double> @splat_concat4(double* %p) { +; CHECK-LABEL: splat_concat4: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastsd (%rdi), %ymm0 +; CHECK-NEXT: retq %1 = load double, double* %p, align 8 %2 = insertelement <2 x double> undef, double %1, i32 0 %3 = insertelement <2 x double> %2, double %1, i32 1 @@ -203,4 +375,3 @@ define <4 x double> @splat_concat4(double* %p) { %6 = shufflevector <2 x double> %3, <2 x double> %5, <4 x i32> ret <4 x double> %6 } - diff --git a/test/CodeGen/X86/avx-vperm2x128.ll b/test/CodeGen/X86/avx-vperm2x128.ll index 4e43f6f51921..0958008d9a3e 100644 --- a/test/CodeGen/X86/avx-vperm2x128.ll +++ b/test/CodeGen/X86/avx-vperm2x128.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX1 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX2 @@ -262,13 +263,13 @@ entry: ret <8 x float> %shuffle } -;; Test zero mask generation. +;; Test zero mask generation. ;; PR22984: https://llvm.org/bugs/show_bug.cgi?id=22984 ;; Prefer xor+vblendpd over vperm2f128 because that has better performance. define <4 x double> @vperm2z_0x08(<4 x double> %a) { ; ALL-LABEL: vperm2z_0x08: -; ALL: # BB#0: +; ALL: ## BB#0: ; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = zero,zero,ymm0[0,1] ; ALL-NEXT: retq %s = shufflevector <4 x double> %a, <4 x double> , <4 x i32> @@ -277,7 +278,7 @@ define <4 x double> @vperm2z_0x08(<4 x double> %a) { define <4 x double> @vperm2z_0x18(<4 x double> %a) { ; ALL-LABEL: vperm2z_0x18: -; ALL: # BB#0: +; ALL: ## BB#0: ; ALL-NEXT: vxorpd %ymm1, %ymm1, %ymm1 ; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3] ; ALL-NEXT: retq @@ -287,7 +288,7 @@ define <4 x double> @vperm2z_0x18(<4 x double> %a) { define <4 x double> @vperm2z_0x28(<4 x double> %a) { ; ALL-LABEL: vperm2z_0x28: -; ALL: # BB#0: +; ALL: ## BB#0: ; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = zero,zero,ymm0[0,1] ; ALL-NEXT: retq %s = shufflevector <4 x double> , <4 x double> %a, <4 x i32> @@ -296,7 +297,7 @@ define <4 x double> @vperm2z_0x28(<4 x double> %a) { define <4 x double> @vperm2z_0x38(<4 x double> %a) { ; ALL-LABEL: vperm2z_0x38: -; ALL: # BB#0: +; ALL: ## BB#0: ; ALL-NEXT: vxorpd %ymm1, %ymm1, %ymm1 ; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3] ; ALL-NEXT: retq @@ -306,8 +307,9 @@ define <4 x double> @vperm2z_0x38(<4 x double> %a) { define <4 x double> @vperm2z_0x80(<4 x double> %a) { ; ALL-LABEL: vperm2z_0x80: -; ALL: # BB#0: -; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],zero,zero +; ALL: ## BB#0: +; ALL-NEXT: vxorpd %ymm1, %ymm1, %ymm1 +; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3] ; ALL-NEXT: retq %s = shufflevector <4 x double> %a, <4 x double> , <4 x i32> ret <4 x double> %s @@ -315,7 +317,7 @@ define <4 x double> @vperm2z_0x80(<4 x double> %a) { define <4 x double> @vperm2z_0x81(<4 x double> %a) { ; ALL-LABEL: vperm2z_0x81: -; ALL: # BB#0: +; ALL: ## BB#0: ; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],zero,zero ; ALL-NEXT: retq %s = shufflevector <4 x double> %a, <4 x double> , <4 x i32> @@ -324,8 +326,9 @@ define <4 x double> @vperm2z_0x81(<4 x double> %a) { define <4 x double> @vperm2z_0x82(<4 x double> %a) { ; ALL-LABEL: vperm2z_0x82: -; ALL: # BB#0: -; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],zero,zero +; ALL: ## BB#0: +; ALL-NEXT: vxorpd %ymm1, %ymm1, %ymm1 +; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3] ; ALL-NEXT: retq %s = shufflevector <4 x double> , <4 x double> %a, <4 x i32> ret <4 x double> %s @@ -333,7 +336,7 @@ define <4 x double> @vperm2z_0x82(<4 x double> %a) { define <4 x double> @vperm2z_0x83(<4 x double> %a) { ; ALL-LABEL: vperm2z_0x83: -; ALL: # BB#0: +; ALL: ## BB#0: ; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],zero,zero ; ALL-NEXT: retq %s = shufflevector <4 x double> , <4 x double> %a, <4 x i32> @@ -343,10 +346,21 @@ define <4 x double> @vperm2z_0x83(<4 x double> %a) { ;; With AVX2 select the integer version of the instruction. Use an add to force the domain selection. define <4 x i64> @vperm2z_int_0x83(<4 x i64> %a, <4 x i64> %b) { -; ALL-LABEL: vperm2z_int_0x83: -; ALL: # BB#0: -; AVX1: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],zero,zero -; AVX2: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],zero,zero +; AVX1-LABEL: vperm2z_int_0x83: +; AVX1: ## BB#0: +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],zero,zero +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3 +; AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpaddq %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: vperm2z_int_0x83: +; AVX2: ## BB#0: +; AVX2-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],zero,zero +; AVX2-NEXT: vpaddq %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq %s = shufflevector <4 x i64> , <4 x i64> %a, <4 x i32> %c = add <4 x i64> %b, %s ret <4 x i64> %c diff --git a/test/CodeGen/X86/avx-win64.ll b/test/CodeGen/X86/avx-win64.ll index dc6bd594450f..64bc398a97ea 100644 --- a/test/CodeGen/X86/avx-win64.ll +++ b/test/CodeGen/X86/avx-win64.ll @@ -42,6 +42,4 @@ safe_if_after_false: ; preds = %safe_if_run_false, } declare i32 @llvm.x86.avx.movmsk.ps.256(<8 x float>) nounwind readnone -declare <8 x float> @llvm.x86.avx.maskload.ps.256(i8*, <8 x float>) nounwind readonly -declare void @llvm.x86.avx.maskstore.ps.256(i8*, <8 x float>, <8 x float>) nounwind declare <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float>, <8 x float>, <8 x float>) nounwind readnone diff --git a/test/CodeGen/X86/avx.ll b/test/CodeGen/X86/avx.ll index f71ec5c10e69..341dd867e4ff 100644 --- a/test/CodeGen/X86/avx.ll +++ b/test/CodeGen/X86/avx.ll @@ -32,7 +32,7 @@ define <4 x float> @insertps_from_vector_load(<4 x float> %a, <4 x float>* nocap ; On X32, account for the argument's move to registers ; X32: movl 4(%esp), %eax ; CHECK-NOT: mov -; CHECK: insertps $48 +; CHECK: vinsertps $48, (%{{...}}), {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] ; CHECK-NEXT: ret %1 = load <4 x float>, <4 x float>* %pb, align 16 %2 = tail call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %a, <4 x float> %1, i32 48) @@ -46,7 +46,7 @@ define <4 x float> @insertps_from_vector_load_offset(<4 x float> %a, <4 x float> ; X32: movl 4(%esp), %eax ; CHECK-NOT: mov ;; Try to match a bit more of the instr, since we need the load's offset. -; CHECK: insertps $96, 4(%{{...}}), % +; CHECK: vinsertps $32, 4(%{{...}}), {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3] ; CHECK-NEXT: ret %1 = load <4 x float>, <4 x float>* %pb, align 16 %2 = tail call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %a, <4 x float> %1, i32 96) @@ -60,7 +60,7 @@ define <4 x float> @insertps_from_vector_load_offset_2(<4 x float> %a, <4 x floa ; X32: movl 8(%esp), %ecx ; CHECK-NOT: mov ;; Try to match a bit more of the instr, since we need the load's offset. -; CHECK: vinsertps $192, 12(%{{...}},%{{...}}), % +; CHECK: vinsertps $0, 12(%{{...}},%{{...}}), {{.*#+}} xmm0 = mem[0],xmm0[1,2,3] ; CHECK-NEXT: ret %1 = getelementptr inbounds <4 x float>, <4 x float>* %pb, i64 %index %2 = load <4 x float>, <4 x float>* %1, align 16 diff --git a/test/CodeGen/X86/avx2-conversions.ll b/test/CodeGen/X86/avx2-conversions.ll index 9b6d5aa5eeae..176292768253 100755 --- a/test/CodeGen/X86/avx2-conversions.ll +++ b/test/CodeGen/X86/avx2-conversions.ll @@ -1,136 +1,151 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=core-avx2 -mattr=+avx2 | FileCheck %s -; CHECK: trunc4 -; CHECK: vpermd -; CHECK-NOT: vinsert -; CHECK: ret define <4 x i32> @trunc4(<4 x i64> %A) nounwind { +; CHECK-LABEL: trunc4: +; CHECK: ## BB#0: +; CHECK-NEXT: vmovdqa {{.*#+}} ymm1 = <0,2,4,6,u,u,u,u> +; CHECK-NEXT: vpermd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq %B = trunc <4 x i64> %A to <4 x i32> ret <4 x i32>%B } -; CHECK: trunc8 -; CHECK: vpshufb -; CHECK-NOT: vinsert -; CHECK: ret - define <8 x i16> @trunc8(<8 x i32> %A) nounwind { +; CHECK-LABEL: trunc8: +; CHECK: ## BB#0: +; CHECK-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,20,21,24,25,28,29],zero,zero,zero,zero,zero,zero,zero,zero +; CHECK-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq %B = trunc <8 x i32> %A to <8 x i16> ret <8 x i16>%B } -; CHECK: sext4 -; CHECK: vpmovsxdq -; CHECK-NOT: vinsert -; CHECK: ret define <4 x i64> @sext4(<4 x i32> %A) nounwind { +; CHECK-LABEL: sext4: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovsxdq %xmm0, %ymm0 +; CHECK-NEXT: retq %B = sext <4 x i32> %A to <4 x i64> ret <4 x i64>%B } -; CHECK: sext8 -; CHECK: vpmovsxwd -; CHECK-NOT: vinsert -; CHECK: ret define <8 x i32> @sext8(<8 x i16> %A) nounwind { +; CHECK-LABEL: sext8: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovsxwd %xmm0, %ymm0 +; CHECK-NEXT: retq %B = sext <8 x i16> %A to <8 x i32> ret <8 x i32>%B } -; CHECK: zext4 -; CHECK: vpmovzxdq -; CHECK-NOT: vinsert -; CHECK: ret define <4 x i64> @zext4(<4 x i32> %A) nounwind { +; CHECK-LABEL: zext4: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; CHECK-NEXT: retq %B = zext <4 x i32> %A to <4 x i64> ret <4 x i64>%B } -; CHECK: zext8 -; CHECK: vpmovzxwd -; CHECK-NOT: vinsert -; CHECK: ret define <8 x i32> @zext8(<8 x i16> %A) nounwind { +; CHECK-LABEL: zext8: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; CHECK-NEXT: retq %B = zext <8 x i16> %A to <8 x i32> ret <8 x i32>%B } -; CHECK: zext_8i8_8i32 -; CHECK: vpmovzxwd -; CHECK: vpand -; CHECK: ret + define <8 x i32> @zext_8i8_8i32(<8 x i8> %A) nounwind { - %B = zext <8 x i8> %A to <8 x i32> +; CHECK-LABEL: zext_8i8_8i32: +; CHECK: ## BB#0: +; CHECK-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; CHECK-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; CHECK-NEXT: retq + %B = zext <8 x i8> %A to <8 x i32> ret <8 x i32>%B } -; CHECK-LABEL: zext_16i8_16i16: -; CHECK: vpmovzxbw -; CHECK-NOT: vinsert -; CHECK: ret define <16 x i16> @zext_16i8_16i16(<16 x i8> %z) { +; CHECK-LABEL: zext_16i8_16i16: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; CHECK-NEXT: retq %t = zext <16 x i8> %z to <16 x i16> ret <16 x i16> %t } -; CHECK-LABEL: sext_16i8_16i16: -; CHECK: vpmovsxbw -; CHECK-NOT: vinsert -; CHECK: ret define <16 x i16> @sext_16i8_16i16(<16 x i8> %z) { +; CHECK-LABEL: sext_16i8_16i16: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovsxbw %xmm0, %ymm0 +; CHECK-NEXT: retq %t = sext <16 x i8> %z to <16 x i16> ret <16 x i16> %t } -; CHECK-LABEL: trunc_16i16_16i8: -; CHECK: vpshufb -; CHECK: vpshufb -; CHECK: vpunpcklqdq -; CHECK: ret define <16 x i8> @trunc_16i16_16i8(<16 x i16> %z) { +; CHECK-LABEL: trunc_16i16_16i8: +; CHECK: ## BB#0: +; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm1 +; CHECK-NEXT: vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u> +; CHECK-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; CHECK-NEXT: vpshufb %xmm2, %xmm0, %xmm0 +; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq %t = trunc <16 x i16> %z to <16 x i8> ret <16 x i8> %t } -; CHECK: load_sext_test1 -; CHECK: vpmovsxdq (%r{{[^,]*}}), %ymm{{.*}} -; CHECK: ret define <4 x i64> @load_sext_test1(<4 x i32> *%ptr) { +; CHECK-LABEL: load_sext_test1: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovsxdq (%rdi), %ymm0 +; CHECK-NEXT: retq %X = load <4 x i32>, <4 x i32>* %ptr %Y = sext <4 x i32> %X to <4 x i64> ret <4 x i64>%Y } -; CHECK: load_sext_test2 -; CHECK: vpmovsxbq (%r{{[^,]*}}), %ymm{{.*}} -; CHECK: ret define <4 x i64> @load_sext_test2(<4 x i8> *%ptr) { +; CHECK-LABEL: load_sext_test2: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovsxbq (%rdi), %ymm0 +; CHECK-NEXT: retq %X = load <4 x i8>, <4 x i8>* %ptr %Y = sext <4 x i8> %X to <4 x i64> ret <4 x i64>%Y } -; CHECK: load_sext_test3 -; CHECK: vpmovsxwq (%r{{[^,]*}}), %ymm{{.*}} -; CHECK: ret define <4 x i64> @load_sext_test3(<4 x i16> *%ptr) { +; CHECK-LABEL: load_sext_test3: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovsxwq (%rdi), %ymm0 +; CHECK-NEXT: retq %X = load <4 x i16>, <4 x i16>* %ptr %Y = sext <4 x i16> %X to <4 x i64> ret <4 x i64>%Y } -; CHECK: load_sext_test4 -; CHECK: vpmovsxwd (%r{{[^,]*}}), %ymm{{.*}} -; CHECK: ret define <8 x i32> @load_sext_test4(<8 x i16> *%ptr) { +; CHECK-LABEL: load_sext_test4: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovsxwd (%rdi), %ymm0 +; CHECK-NEXT: retq %X = load <8 x i16>, <8 x i16>* %ptr %Y = sext <8 x i16> %X to <8 x i32> ret <8 x i32>%Y } -; CHECK: load_sext_test5 -; CHECK: vpmovsxbd (%r{{[^,]*}}), %ymm{{.*}} -; CHECK: ret define <8 x i32> @load_sext_test5(<8 x i8> *%ptr) { +; CHECK-LABEL: load_sext_test5: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovsxbd (%rdi), %ymm0 +; CHECK-NEXT: retq %X = load <8 x i8>, <8 x i8>* %ptr %Y = sext <8 x i8> %X to <8 x i32> ret <8 x i32>%Y diff --git a/test/CodeGen/X86/avx2-intrinsics-x86-upgrade.ll b/test/CodeGen/X86/avx2-intrinsics-x86-upgrade.ll index a30d8371775c..36b6da5ef960 100644 --- a/test/CodeGen/X86/avx2-intrinsics-x86-upgrade.ll +++ b/test/CodeGen/X86/avx2-intrinsics-x86-upgrade.ll @@ -83,3 +83,123 @@ define <4 x i64> @test_x86_avx2_vinserti128(<4 x i64> %a0, <2 x i64> %a1) { } declare <4 x i64> @llvm.x86.avx2.vinserti128(<4 x i64>, <2 x i64>, i8) nounwind readnone + +define <4 x double> @test_x86_avx2_vbroadcast_sd_pd_256(<2 x double> %a0) { + ; CHECK-LABEL: test_x86_avx2_vbroadcast_sd_pd_256: + ; CHECK: ## BB#0: + ; CHECK-NEXT: vbroadcastsd %xmm0, %ymm0 + ; CHECK-NEXT: retl + %res = call <4 x double> @llvm.x86.avx2.vbroadcast.sd.pd.256(<2 x double> %a0) + ret <4 x double> %res +} +declare <4 x double> @llvm.x86.avx2.vbroadcast.sd.pd.256(<2 x double>) nounwind readonly + + +define <4 x float> @test_x86_avx2_vbroadcast_ss_ps(<4 x float> %a0) { + ; CHECK-LABEL: test_x86_avx2_vbroadcast_ss_ps: + ; CHECK: ## BB#0: + ; CHECK-NEXT: vbroadcastss %xmm0, %xmm0 + ; CHECK-NEXT: retl + %res = call <4 x float> @llvm.x86.avx2.vbroadcast.ss.ps(<4 x float> %a0) + ret <4 x float> %res +} +declare <4 x float> @llvm.x86.avx2.vbroadcast.ss.ps(<4 x float>) nounwind readonly + + +define <8 x float> @test_x86_avx2_vbroadcast_ss_ps_256(<4 x float> %a0) { + ; CHECK-LABEL: test_x86_avx2_vbroadcast_ss_ps_256: + ; CHECK: ## BB#0: + ; CHECK-NEXT: vbroadcastss %xmm0, %ymm0 + ; CHECK-NEXT: retl + %res = call <8 x float> @llvm.x86.avx2.vbroadcast.ss.ps.256(<4 x float> %a0) + ret <8 x float> %res +} +declare <8 x float> @llvm.x86.avx2.vbroadcast.ss.ps.256(<4 x float>) nounwind readonly + + +define <16 x i8> @test_x86_avx2_pbroadcastb_128(<16 x i8> %a0) { +; CHECK-LABEL: test_x86_avx2_pbroadcastb_128: +; CHECK: ## BB#0: +; CHECK-NEXT: vpbroadcastb %xmm0, %xmm0 +; CHECK-NEXT: retl + %res = call <16 x i8> @llvm.x86.avx2.pbroadcastb.128(<16 x i8> %a0) + ret <16 x i8> %res +} +declare <16 x i8> @llvm.x86.avx2.pbroadcastb.128(<16 x i8>) nounwind readonly + + +define <32 x i8> @test_x86_avx2_pbroadcastb_256(<16 x i8> %a0) { +; CHECK-LABEL: test_x86_avx2_pbroadcastb_256: +; CHECK: ## BB#0: +; CHECK-NEXT: vpbroadcastb %xmm0, %ymm0 +; CHECK-NEXT: retl + %res = call <32 x i8> @llvm.x86.avx2.pbroadcastb.256(<16 x i8> %a0) + ret <32 x i8> %res +} +declare <32 x i8> @llvm.x86.avx2.pbroadcastb.256(<16 x i8>) nounwind readonly + + +define <8 x i16> @test_x86_avx2_pbroadcastw_128(<8 x i16> %a0) { +; CHECK-LABEL: test_x86_avx2_pbroadcastw_128: +; CHECK: ## BB#0: +; CHECK-NEXT: vpbroadcastw %xmm0, %xmm0 +; CHECK-NEXT: retl + %res = call <8 x i16> @llvm.x86.avx2.pbroadcastw.128(<8 x i16> %a0) + ret <8 x i16> %res +} +declare <8 x i16> @llvm.x86.avx2.pbroadcastw.128(<8 x i16>) nounwind readonly + + +define <16 x i16> @test_x86_avx2_pbroadcastw_256(<8 x i16> %a0) { +; CHECK-LABEL: test_x86_avx2_pbroadcastw_256: +; CHECK: ## BB#0: +; CHECK-NEXT: vpbroadcastw %xmm0, %ymm0 +; CHECK-NEXT: retl + %res = call <16 x i16> @llvm.x86.avx2.pbroadcastw.256(<8 x i16> %a0) + ret <16 x i16> %res +} +declare <16 x i16> @llvm.x86.avx2.pbroadcastw.256(<8 x i16>) nounwind readonly + + +define <4 x i32> @test_x86_avx2_pbroadcastd_128(<4 x i32> %a0) { +; CHECK-LABEL: test_x86_avx2_pbroadcastd_128: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastss %xmm0, %xmm0 +; CHECK-NEXT: retl + %res = call <4 x i32> @llvm.x86.avx2.pbroadcastd.128(<4 x i32> %a0) + ret <4 x i32> %res +} +declare <4 x i32> @llvm.x86.avx2.pbroadcastd.128(<4 x i32>) nounwind readonly + + +define <8 x i32> @test_x86_avx2_pbroadcastd_256(<4 x i32> %a0) { +; CHECK-LABEL: test_x86_avx2_pbroadcastd_256: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastss %xmm0, %ymm0 +; CHECK-NEXT: retl + %res = call <8 x i32> @llvm.x86.avx2.pbroadcastd.256(<4 x i32> %a0) + ret <8 x i32> %res +} +declare <8 x i32> @llvm.x86.avx2.pbroadcastd.256(<4 x i32>) nounwind readonly + + +define <2 x i64> @test_x86_avx2_pbroadcastq_128(<2 x i64> %a0) { +; CHECK-LABEL: test_x86_avx2_pbroadcastq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: vpbroadcastq %xmm0, %xmm0 +; CHECK-NEXT: retl + %res = call <2 x i64> @llvm.x86.avx2.pbroadcastq.128(<2 x i64> %a0) + ret <2 x i64> %res +} +declare <2 x i64> @llvm.x86.avx2.pbroadcastq.128(<2 x i64>) nounwind readonly + + +define <4 x i64> @test_x86_avx2_pbroadcastq_256(<2 x i64> %a0) { +; CHECK-LABEL: test_x86_avx2_pbroadcastq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastsd %xmm0, %ymm0 +; CHECK-NEXT: retl + %res = call <4 x i64> @llvm.x86.avx2.pbroadcastq.256(<2 x i64> %a0) + ret <4 x i64> %res +} +declare <4 x i64> @llvm.x86.avx2.pbroadcastq.256(<2 x i64>) nounwind readonly diff --git a/test/CodeGen/X86/avx2-intrinsics-x86.ll b/test/CodeGen/X86/avx2-intrinsics-x86.ll index 5b607afef91c..606aca9dc02b 100644 --- a/test/CodeGen/X86/avx2-intrinsics-x86.ll +++ b/test/CodeGen/X86/avx2-intrinsics-x86.ll @@ -641,30 +641,6 @@ define <4 x i64> @test_x86_avx2_pmul.dq(<8 x i32> %a0, <8 x i32> %a1) { declare <4 x i64> @llvm.x86.avx2.pmul.dq(<8 x i32>, <8 x i32>) nounwind readnone -define <4 x double> @test_x86_avx2_vbroadcast_sd_pd_256(<2 x double> %a0) { - ; CHECK: vbroadcastsd - %res = call <4 x double> @llvm.x86.avx2.vbroadcast.sd.pd.256(<2 x double> %a0) ; <<4 x double>> [#uses=1] - ret <4 x double> %res -} -declare <4 x double> @llvm.x86.avx2.vbroadcast.sd.pd.256(<2 x double>) nounwind readonly - - -define <4 x float> @test_x86_avx2_vbroadcast_ss_ps(<4 x float> %a0) { - ; CHECK: vbroadcastss - %res = call <4 x float> @llvm.x86.avx2.vbroadcast.ss.ps(<4 x float> %a0) ; <<4 x float>> [#uses=1] - ret <4 x float> %res -} -declare <4 x float> @llvm.x86.avx2.vbroadcast.ss.ps(<4 x float>) nounwind readonly - - -define <8 x float> @test_x86_avx2_vbroadcast_ss_ps_256(<4 x float> %a0) { - ; CHECK: vbroadcastss - %res = call <8 x float> @llvm.x86.avx2.vbroadcast.ss.ps.256(<4 x float> %a0) ; <<8 x float>> [#uses=1] - ret <8 x float> %res -} -declare <8 x float> @llvm.x86.avx2.vbroadcast.ss.ps.256(<4 x float>) nounwind readonly - - define <4 x i32> @test_x86_avx2_pblendd_128(<4 x i32> %a0, <4 x i32> %a1) { ; CHECK: vpblendd %res = call <4 x i32> @llvm.x86.avx2.pblendd.128(<4 x i32> %a0, <4 x i32> %a1, i8 7) ; <<4 x i32>> [#uses=1] @@ -681,70 +657,6 @@ define <8 x i32> @test_x86_avx2_pblendd_256(<8 x i32> %a0, <8 x i32> %a1) { declare <8 x i32> @llvm.x86.avx2.pblendd.256(<8 x i32>, <8 x i32>, i8) nounwind readnone -define <16 x i8> @test_x86_avx2_pbroadcastb_128(<16 x i8> %a0) { - ; CHECK: vpbroadcastb - %res = call <16 x i8> @llvm.x86.avx2.pbroadcastb.128(<16 x i8> %a0) ; <<16 x i8>> [#uses=1] - ret <16 x i8> %res -} -declare <16 x i8> @llvm.x86.avx2.pbroadcastb.128(<16 x i8>) nounwind readonly - - -define <32 x i8> @test_x86_avx2_pbroadcastb_256(<16 x i8> %a0) { - ; CHECK: vpbroadcastb - %res = call <32 x i8> @llvm.x86.avx2.pbroadcastb.256(<16 x i8> %a0) ; <<32 x i8>> [#uses=1] - ret <32 x i8> %res -} -declare <32 x i8> @llvm.x86.avx2.pbroadcastb.256(<16 x i8>) nounwind readonly - - -define <8 x i16> @test_x86_avx2_pbroadcastw_128(<8 x i16> %a0) { - ; CHECK: vpbroadcastw - %res = call <8 x i16> @llvm.x86.avx2.pbroadcastw.128(<8 x i16> %a0) ; <<8 x i16>> [#uses=1] - ret <8 x i16> %res -} -declare <8 x i16> @llvm.x86.avx2.pbroadcastw.128(<8 x i16>) nounwind readonly - - -define <16 x i16> @test_x86_avx2_pbroadcastw_256(<8 x i16> %a0) { - ; CHECK: vpbroadcastw - %res = call <16 x i16> @llvm.x86.avx2.pbroadcastw.256(<8 x i16> %a0) ; <<16 x i16>> [#uses=1] - ret <16 x i16> %res -} -declare <16 x i16> @llvm.x86.avx2.pbroadcastw.256(<8 x i16>) nounwind readonly - - -define <4 x i32> @test_x86_avx2_pbroadcastd_128(<4 x i32> %a0) { - ; CHECK: vbroadcastss - %res = call <4 x i32> @llvm.x86.avx2.pbroadcastd.128(<4 x i32> %a0) ; <<4 x i32>> [#uses=1] - ret <4 x i32> %res -} -declare <4 x i32> @llvm.x86.avx2.pbroadcastd.128(<4 x i32>) nounwind readonly - - -define <8 x i32> @test_x86_avx2_pbroadcastd_256(<4 x i32> %a0) { - ; CHECK: vbroadcastss {{[^,]+}}, %ymm{{[0-9]+}} - %res = call <8 x i32> @llvm.x86.avx2.pbroadcastd.256(<4 x i32> %a0) ; <<8 x i32>> [#uses=1] - ret <8 x i32> %res -} -declare <8 x i32> @llvm.x86.avx2.pbroadcastd.256(<4 x i32>) nounwind readonly - - -define <2 x i64> @test_x86_avx2_pbroadcastq_128(<2 x i64> %a0) { - ; CHECK: vpbroadcastq - %res = call <2 x i64> @llvm.x86.avx2.pbroadcastq.128(<2 x i64> %a0) ; <<2 x i64>> [#uses=1] - ret <2 x i64> %res -} -declare <2 x i64> @llvm.x86.avx2.pbroadcastq.128(<2 x i64>) nounwind readonly - - -define <4 x i64> @test_x86_avx2_pbroadcastq_256(<2 x i64> %a0) { - ; CHECK: vbroadcastsd {{[^,]+}}, %ymm{{[0-9]+}} - %res = call <4 x i64> @llvm.x86.avx2.pbroadcastq.256(<2 x i64> %a0) ; <<4 x i64>> [#uses=1] - ret <4 x i64> %res -} -declare <4 x i64> @llvm.x86.avx2.pbroadcastq.256(<2 x i64>) nounwind readonly - - define <8 x i32> @test_x86_avx2_permd(<8 x i32> %a0, <8 x i32> %a1) { ; Check that the arguments are swapped between the intrinsic definition ; and its lowering. Indeed, the offsets are the first source in @@ -756,15 +668,15 @@ define <8 x i32> @test_x86_avx2_permd(<8 x i32> %a0, <8 x i32> %a1) { declare <8 x i32> @llvm.x86.avx2.permd(<8 x i32>, <8 x i32>) nounwind readonly -define <8 x float> @test_x86_avx2_permps(<8 x float> %a0, <8 x float> %a1) { +define <8 x float> @test_x86_avx2_permps(<8 x float> %a0, <8 x i32> %a1) { ; Check that the arguments are swapped between the intrinsic definition ; and its lowering. Indeed, the offsets are the first source in ; the instruction. ; CHECK: vpermps %ymm0, %ymm1, %ymm0 - %res = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x float> %a1) ; <<8 x float>> [#uses=1] + %res = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> %a1) ; <<8 x float>> [#uses=1] ret <8 x float> %res } -declare <8 x float> @llvm.x86.avx2.permps(<8 x float>, <8 x float>) nounwind readonly +declare <8 x float> @llvm.x86.avx2.permps(<8 x float>, <8 x i32>) nounwind readonly define <4 x i64> @test_x86_avx2_vperm2i128(<4 x i64> %a0, <4 x i64> %a1) { diff --git a/test/CodeGen/X86/avx2-nontemporal.ll b/test/CodeGen/X86/avx2-nontemporal.ll index 4d28a979712a..058358f13b86 100644 --- a/test/CodeGen/X86/avx2-nontemporal.ll +++ b/test/CodeGen/X86/avx2-nontemporal.ll @@ -1,21 +1,18 @@ ; RUN: llc < %s -march=x86 -mattr=+avx2 | FileCheck %s -define void @f(<8 x float> %A, i8* %B, <4 x double> %C, i32 %D, <4 x i64> %E) { -; CHECK: vmovntps +define void @f(<8 x float> %A, i8* %B, <4 x double> %C, <4 x i64> %E) { +; CHECK: vmovntps %y %cast = bitcast i8* %B to <8 x float>* %A2 = fadd <8 x float> %A, - store <8 x float> %A2, <8 x float>* %cast, align 16, !nontemporal !0 -; CHECK: vmovntdq + store <8 x float> %A2, <8 x float>* %cast, align 32, !nontemporal !0 +; CHECK: vmovntdq %y %cast1 = bitcast i8* %B to <4 x i64>* %E2 = add <4 x i64> %E, - store <4 x i64> %E2, <4 x i64>* %cast1, align 16, !nontemporal !0 -; CHECK: vmovntpd + store <4 x i64> %E2, <4 x i64>* %cast1, align 32, !nontemporal !0 +; CHECK: vmovntpd %y %cast2 = bitcast i8* %B to <4 x double>* %C2 = fadd <4 x double> %C, - store <4 x double> %C2, <4 x double>* %cast2, align 16, !nontemporal !0 -; CHECK: movnti - %cast3 = bitcast i8* %B to i32* - store i32 %D, i32* %cast3, align 16, !nontemporal !0 + store <4 x double> %C2, <4 x double>* %cast2, align 32, !nontemporal !0 ret void } diff --git a/test/CodeGen/X86/avx2-vbroadcast.ll b/test/CodeGen/X86/avx2-vbroadcast.ll index 94dcdcabdd33..6b77edb155a4 100644 --- a/test/CodeGen/X86/avx2-vbroadcast.ll +++ b/test/CodeGen/X86/avx2-vbroadcast.ll @@ -1,7 +1,11 @@ -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=core-avx2 -mattr=+avx2 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx2 | FileCheck %s -; CHECK: vpbroadcastb (% define <16 x i8> @BB16(i8* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: BB16: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpbroadcastb (%rdi), %xmm0 +; CHECK-NEXT: retq entry: %q = load i8, i8* %ptr, align 4 %q0 = insertelement <16 x i8> undef, i8 %q, i32 0 @@ -22,8 +26,12 @@ entry: %qf = insertelement <16 x i8> %qe, i8 %q, i32 15 ret <16 x i8> %qf } -; CHECK: vpbroadcastb (% + define <32 x i8> @BB32(i8* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: BB32: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpbroadcastb (%rdi), %ymm0 +; CHECK-NEXT: retq entry: %q = load i8, i8* %ptr, align 4 %q0 = insertelement <32 x i8> undef, i8 %q, i32 0 @@ -61,9 +69,12 @@ entry: %q2f = insertelement <32 x i8> %q2e, i8 %q, i32 31 ret <32 x i8> %q2f } -; CHECK: vpbroadcastw (% define <8 x i16> @W16(i16* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: W16: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpbroadcastw (%rdi), %xmm0 +; CHECK-NEXT: retq entry: %q = load i16, i16* %ptr, align 4 %q0 = insertelement <8 x i16> undef, i16 %q, i32 0 @@ -76,8 +87,12 @@ entry: %q7 = insertelement <8 x i16> %q6, i16 %q, i32 7 ret <8 x i16> %q7 } -; CHECK: vpbroadcastw (% + define <16 x i16> @WW16(i16* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: WW16: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpbroadcastw (%rdi), %ymm0 +; CHECK-NEXT: retq entry: %q = load i16, i16* %ptr, align 4 %q0 = insertelement <16 x i16> undef, i16 %q, i32 0 @@ -98,8 +113,12 @@ entry: %qf = insertelement <16 x i16> %qe, i16 %q, i32 15 ret <16 x i16> %qf } -; CHECK: vbroadcastss (% + define <4 x i32> @D32(i32* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: D32: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss (%rdi), %xmm0 +; CHECK-NEXT: retq entry: %q = load i32, i32* %ptr, align 4 %q0 = insertelement <4 x i32> undef, i32 %q, i32 0 @@ -108,8 +127,12 @@ entry: %q3 = insertelement <4 x i32> %q2, i32 %q, i32 3 ret <4 x i32> %q3 } -; CHECK: vbroadcastss (% + define <8 x i32> @DD32(i32* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: DD32: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss (%rdi), %ymm0 +; CHECK-NEXT: retq entry: %q = load i32, i32* %ptr, align 4 %q0 = insertelement <8 x i32> undef, i32 %q, i32 0 @@ -122,16 +145,24 @@ entry: %q7 = insertelement <8 x i32> %q6, i32 %q, i32 7 ret <8 x i32> %q7 } -; CHECK: vpbroadcastq (% + define <2 x i64> @Q64(i64* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: Q64: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpbroadcastq (%rdi), %xmm0 +; CHECK-NEXT: retq entry: %q = load i64, i64* %ptr, align 4 %q0 = insertelement <2 x i64> undef, i64 %q, i32 0 %q1 = insertelement <2 x i64> %q0, i64 %q, i32 1 ret <2 x i64> %q1 } -; CHECK: vbroadcastsd (% + define <4 x i64> @QQ64(i64* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: QQ64: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastsd (%rdi), %ymm0 +; CHECK-NEXT: retq entry: %q = load i64, i64* %ptr, align 4 %q0 = insertelement <4 x i64> undef, i64 %q, i32 0 @@ -141,9 +172,214 @@ entry: ret <4 x i64> %q3 } +; FIXME: Pointer adjusted broadcasts + +define <16 x i8> @load_splat_16i8_16i8_1111111111111111(<16 x i8>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_16i8_16i8_1111111111111111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpbroadcastb 1(%rdi), %xmm0 +; CHECK-NEXT: retq +entry: + %ld = load <16 x i8>, <16 x i8>* %ptr + %ret = shufflevector <16 x i8> %ld, <16 x i8> undef, <16 x i32> + ret <16 x i8> %ret +} + +define <32 x i8> @load_splat_32i8_16i8_11111111111111111111111111111111(<16 x i8>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_32i8_16i8_11111111111111111111111111111111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpbroadcastb 1(%rdi), %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <16 x i8>, <16 x i8>* %ptr + %ret = shufflevector <16 x i8> %ld, <16 x i8> undef, <32 x i32> + ret <32 x i8> %ret +} + +define <32 x i8> @load_splat_32i8_32i8_11111111111111111111111111111111(<32 x i8>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_32i8_32i8_11111111111111111111111111111111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpbroadcastb 1(%rdi), %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <32 x i8>, <32 x i8>* %ptr + %ret = shufflevector <32 x i8> %ld, <32 x i8> undef, <32 x i32> + ret <32 x i8> %ret +} + +define <8 x i16> @load_splat_8i16_8i16_11111111(<8 x i16>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_8i16_8i16_11111111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpbroadcastw 2(%rdi), %xmm0 +; CHECK-NEXT: retq +entry: + %ld = load <8 x i16>, <8 x i16>* %ptr + %ret = shufflevector <8 x i16> %ld, <8 x i16> undef, <8 x i32> + ret <8 x i16> %ret +} + +define <16 x i16> @load_splat_16i16_8i16_1111111111111111(<8 x i16>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_16i16_8i16_1111111111111111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpbroadcastw 2(%rdi), %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <8 x i16>, <8 x i16>* %ptr + %ret = shufflevector <8 x i16> %ld, <8 x i16> undef, <16 x i32> + ret <16 x i16> %ret +} + +define <16 x i16> @load_splat_16i16_16i16_1111111111111111(<16 x i16>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_16i16_16i16_1111111111111111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpbroadcastw 2(%rdi), %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <16 x i16>, <16 x i16>* %ptr + %ret = shufflevector <16 x i16> %ld, <16 x i16> undef, <16 x i32> + ret <16 x i16> %ret +} + +define <4 x i32> @load_splat_4i32_4i32_1111(<4 x i32>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_4i32_4i32_1111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss 4(%rdi), %xmm0 +; CHECK-NEXT: retq +entry: + %ld = load <4 x i32>, <4 x i32>* %ptr + %ret = shufflevector <4 x i32> %ld, <4 x i32> undef, <4 x i32> + ret <4 x i32> %ret +} + +define <8 x i32> @load_splat_8i32_4i32_33333333(<4 x i32>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_8i32_4i32_33333333: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss 12(%rdi), %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <4 x i32>, <4 x i32>* %ptr + %ret = shufflevector <4 x i32> %ld, <4 x i32> undef, <8 x i32> + ret <8 x i32> %ret +} + +define <8 x i32> @load_splat_8i32_8i32_55555555(<8 x i32>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_8i32_8i32_55555555: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss 20(%rdi), %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <8 x i32>, <8 x i32>* %ptr + %ret = shufflevector <8 x i32> %ld, <8 x i32> undef, <8 x i32> + ret <8 x i32> %ret +} + +define <4 x float> @load_splat_4f32_4f32_1111(<4 x float>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_4f32_4f32_1111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss 4(%rdi), %xmm0 +; CHECK-NEXT: retq +entry: + %ld = load <4 x float>, <4 x float>* %ptr + %ret = shufflevector <4 x float> %ld, <4 x float> undef, <4 x i32> + ret <4 x float> %ret +} + +define <8 x float> @load_splat_8f32_4f32_33333333(<4 x float>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_8f32_4f32_33333333: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss 12(%rdi), %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <4 x float>, <4 x float>* %ptr + %ret = shufflevector <4 x float> %ld, <4 x float> undef, <8 x i32> + ret <8 x float> %ret +} + +define <8 x float> @load_splat_8f32_8f32_55555555(<8 x float>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_8f32_8f32_55555555: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss 20(%rdi), %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <8 x float>, <8 x float>* %ptr + %ret = shufflevector <8 x float> %ld, <8 x float> undef, <8 x i32> + ret <8 x float> %ret +} + +define <2 x i64> @load_splat_2i64_2i64_1111(<2 x i64>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_2i64_2i64_1111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpbroadcastq 8(%rdi), %xmm0 +; CHECK-NEXT: retq +entry: + %ld = load <2 x i64>, <2 x i64>* %ptr + %ret = shufflevector <2 x i64> %ld, <2 x i64> undef, <2 x i32> + ret <2 x i64> %ret +} + +define <4 x i64> @load_splat_4i64_2i64_1111(<2 x i64>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_4i64_2i64_1111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastsd 8(%rdi), %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <2 x i64>, <2 x i64>* %ptr + %ret = shufflevector <2 x i64> %ld, <2 x i64> undef, <4 x i32> + ret <4 x i64> %ret +} + +define <4 x i64> @load_splat_4i64_4i64_2222(<4 x i64>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_4i64_4i64_2222: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastsd 16(%rdi), %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <4 x i64>, <4 x i64>* %ptr + %ret = shufflevector <4 x i64> %ld, <4 x i64> undef, <4 x i32> + ret <4 x i64> %ret +} + +define <2 x double> @load_splat_2f64_2f64_1111(<2 x double>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_2f64_2f64_1111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vmovaps (%rdi), %xmm0 +; CHECK-NEXT: vmovhlps {{.*#+}} xmm0 = xmm0[1,1] +; CHECK-NEXT: retq +entry: + %ld = load <2 x double>, <2 x double>* %ptr + %ret = shufflevector <2 x double> %ld, <2 x double> undef, <2 x i32> + ret <2 x double> %ret +} + +define <4 x double> @load_splat_4f64_2f64_1111(<2 x double>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_4f64_2f64_1111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastsd 8(%rdi), %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <2 x double>, <2 x double>* %ptr + %ret = shufflevector <2 x double> %ld, <2 x double> undef, <4 x i32> + ret <4 x double> %ret +} + +define <4 x double> @load_splat_4f64_4f64_2222(<4 x double>* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: load_splat_4f64_4f64_2222: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastsd 16(%rdi), %ymm0 +; CHECK-NEXT: retq +entry: + %ld = load <4 x double>, <4 x double>* %ptr + %ret = shufflevector <4 x double> %ld, <4 x double> undef, <4 x i32> + ret <4 x double> %ret +} + ; make sure that we still don't support broadcast double into 128-bit vector ; this used to crash define <2 x double> @I(double* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: I: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = mem[0,0] +; CHECK-NEXT: retq entry: %q = load double, double* %ptr, align 4 %vecinit.i = insertelement <2 x double> undef, double %q, i32 0 @@ -151,28 +387,33 @@ entry: ret <2 x double> %vecinit2.i } -; CHECK: V111 -; CHECK: vpbroadcastd -; CHECK: ret define <8 x i32> @V111(<8 x i32> %in) nounwind uwtable readnone ssp { +; CHECK-LABEL: V111: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vpbroadcastd {{.*}}(%rip), %ymm1 +; CHECK-NEXT: vpaddd %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: retq entry: %g = add <8 x i32> %in, ret <8 x i32> %g } -; CHECK: V113 -; CHECK: vbroadcastss -; CHECK: ret define <8 x float> @V113(<8 x float> %in) nounwind uwtable readnone ssp { +; CHECK-LABEL: V113: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: vbroadcastss {{.*}}(%rip), %ymm1 +; CHECK-NEXT: vaddps %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: retq entry: %g = fadd <8 x float> %in, ret <8 x float> %g } -; CHECK: _e2 -; CHECK: vbroadcastss -; CHECK: ret define <4 x float> @_e2(float* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: _e2: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastss {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq %vecinit.i = insertelement <4 x float> undef, float 0xbf80000000000000, i32 0 %vecinit2.i = insertelement <4 x float> %vecinit.i, float 0xbf80000000000000, i32 1 %vecinit4.i = insertelement <4 x float> %vecinit2.i, float 0xbf80000000000000, i32 2 @@ -180,10 +421,11 @@ define <4 x float> @_e2(float* %ptr) nounwind uwtable readnone ssp { ret <4 x float> %vecinit6.i } -; CHECK: _e4 -; CHECK-NOT: broadcast -; CHECK: ret define <8 x i8> @_e4(i8* %ptr) nounwind uwtable readnone ssp { +; CHECK-LABEL: _e4: +; CHECK: ## BB#0: +; CHECK-NEXT: vmovaps {{.*#+}} xmm0 = [52,52,52,52,52,52,52,52] +; CHECK-NEXT: retq %vecinit0.i = insertelement <8 x i8> undef, i8 52, i32 0 %vecinit1.i = insertelement <8 x i8> %vecinit0.i, i8 52, i32 1 %vecinit2.i = insertelement <8 x i8> %vecinit1.i, i8 52, i32 2 @@ -197,6 +439,17 @@ define <8 x i8> @_e4(i8* %ptr) nounwind uwtable readnone ssp { define void @crash() nounwind alwaysinline { +; CHECK-LABEL: crash: +; CHECK: ## BB#0: ## %WGLoopsEntry +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: testb %al, %al +; CHECK-NEXT: je LBB31_1 +; CHECK-NEXT: ## BB#2: ## %ret +; CHECK-NEXT: retq +; CHECK-NEXT: .align 4, 0x90 +; CHECK-NEXT: LBB31_1: ## %footer349VF +; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: jmp LBB31_1 WGLoopsEntry: br i1 undef, label %ret, label %footer329VF @@ -223,135 +476,151 @@ ret: ret void } -; CHECK: _inreg0 -; CHECK: broadcastss -; CHECK: ret define <8 x i32> @_inreg0(i32 %scalar) nounwind uwtable readnone ssp { +; CHECK-LABEL: _inreg0: +; CHECK: ## BB#0: +; CHECK-NEXT: vmovd %edi, %xmm0 +; CHECK-NEXT: vbroadcastss %xmm0, %ymm0 +; CHECK-NEXT: retq %in = insertelement <8 x i32> undef, i32 %scalar, i32 0 %wide = shufflevector <8 x i32> %in, <8 x i32> undef, <8 x i32> zeroinitializer ret <8 x i32> %wide } -; CHECK: _inreg1 -; CHECK: broadcastss -; CHECK: ret define <8 x float> @_inreg1(float %scalar) nounwind uwtable readnone ssp { +; CHECK-LABEL: _inreg1: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastss %xmm0, %ymm0 +; CHECK-NEXT: retq %in = insertelement <8 x float> undef, float %scalar, i32 0 %wide = shufflevector <8 x float> %in, <8 x float> undef, <8 x i32> zeroinitializer ret <8 x float> %wide } -; CHECK: _inreg2 -; CHECK: broadcastss -; CHECK: ret define <4 x float> @_inreg2(float %scalar) nounwind uwtable readnone ssp { +; CHECK-LABEL: _inreg2: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastss %xmm0, %xmm0 +; CHECK-NEXT: retq %in = insertelement <4 x float> undef, float %scalar, i32 0 %wide = shufflevector <4 x float> %in, <4 x float> undef, <4 x i32> zeroinitializer ret <4 x float> %wide } -; CHECK: _inreg3 -; CHECK: broadcastsd -; CHECK: ret define <4 x double> @_inreg3(double %scalar) nounwind uwtable readnone ssp { +; CHECK-LABEL: _inreg3: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastsd %xmm0, %ymm0 +; CHECK-NEXT: retq %in = insertelement <4 x double> undef, double %scalar, i32 0 %wide = shufflevector <4 x double> %in, <4 x double> undef, <4 x i32> zeroinitializer ret <4 x double> %wide } -;CHECK-LABEL: _inreg8xfloat: -;CHECK: vbroadcastss -;CHECK: ret define <8 x float> @_inreg8xfloat(<8 x float> %a) { +; CHECK-LABEL: _inreg8xfloat: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastss %xmm0, %ymm0 +; CHECK-NEXT: retq %b = shufflevector <8 x float> %a, <8 x float> undef, <8 x i32> zeroinitializer ret <8 x float> %b } -;CHECK-LABEL: _inreg4xfloat: -;CHECK: vbroadcastss -;CHECK: ret define <4 x float> @_inreg4xfloat(<4 x float> %a) { +; CHECK-LABEL: _inreg4xfloat: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastss %xmm0, %xmm0 +; CHECK-NEXT: retq %b = shufflevector <4 x float> %a, <4 x float> undef, <4 x i32> zeroinitializer ret <4 x float> %b } -;CHECK-LABEL: _inreg16xi16: -;CHECK: vpbroadcastw -;CHECK: ret define <16 x i16> @_inreg16xi16(<16 x i16> %a) { +; CHECK-LABEL: _inreg16xi16: +; CHECK: ## BB#0: +; CHECK-NEXT: vpbroadcastw %xmm0, %ymm0 +; CHECK-NEXT: retq %b = shufflevector <16 x i16> %a, <16 x i16> undef, <16 x i32> zeroinitializer ret <16 x i16> %b } -;CHECK-LABEL: _inreg8xi16: -;CHECK: vpbroadcastw -;CHECK: ret define <8 x i16> @_inreg8xi16(<8 x i16> %a) { +; CHECK-LABEL: _inreg8xi16: +; CHECK: ## BB#0: +; CHECK-NEXT: vpbroadcastw %xmm0, %xmm0 +; CHECK-NEXT: retq %b = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> zeroinitializer ret <8 x i16> %b } - -;CHECK-LABEL: _inreg4xi64: -;CHECK: vbroadcastsd -;CHECK: ret define <4 x i64> @_inreg4xi64(<4 x i64> %a) { +; CHECK-LABEL: _inreg4xi64: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastsd %xmm0, %ymm0 +; CHECK-NEXT: retq %b = shufflevector <4 x i64> %a, <4 x i64> undef, <4 x i32> zeroinitializer ret <4 x i64> %b } -;CHECK-LABEL: _inreg2xi64: -;CHECK: vpbroadcastq -;CHECK: ret define <2 x i64> @_inreg2xi64(<2 x i64> %a) { +; CHECK-LABEL: _inreg2xi64: +; CHECK: ## BB#0: +; CHECK-NEXT: vpbroadcastq %xmm0, %xmm0 +; CHECK-NEXT: retq %b = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> zeroinitializer ret <2 x i64> %b } -;CHECK-LABEL: _inreg4xdouble: -;CHECK: vbroadcastsd -;CHECK: ret define <4 x double> @_inreg4xdouble(<4 x double> %a) { +; CHECK-LABEL: _inreg4xdouble: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastsd %xmm0, %ymm0 +; CHECK-NEXT: retq %b = shufflevector <4 x double> %a, <4 x double> undef, <4 x i32> zeroinitializer ret <4 x double> %b } -;CHECK-LABEL: _inreg2xdouble: -;CHECK: vmovddup -;CHECK: ret define <2 x double> @_inreg2xdouble(<2 x double> %a) { +; CHECK-LABEL: _inreg2xdouble: +; CHECK: ## BB#0: +; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0] +; CHECK-NEXT: retq %b = shufflevector <2 x double> %a, <2 x double> undef, <2 x i32> zeroinitializer ret <2 x double> %b } -;CHECK-LABEL: _inreg8xi32: -;CHECK: vbroadcastss -;CHECK: ret define <8 x i32> @_inreg8xi32(<8 x i32> %a) { +; CHECK-LABEL: _inreg8xi32: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastss %xmm0, %ymm0 +; CHECK-NEXT: retq %b = shufflevector <8 x i32> %a, <8 x i32> undef, <8 x i32> zeroinitializer ret <8 x i32> %b } -;CHECK-LABEL: _inreg4xi32: -;CHECK: vbroadcastss -;CHECK: ret define <4 x i32> @_inreg4xi32(<4 x i32> %a) { +; CHECK-LABEL: _inreg4xi32: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastss %xmm0, %xmm0 +; CHECK-NEXT: retq %b = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> zeroinitializer ret <4 x i32> %b } -;CHECK-LABEL: _inreg32xi8: -;CHECK: vpbroadcastb -;CHECK: ret define <32 x i8> @_inreg32xi8(<32 x i8> %a) { +; CHECK-LABEL: _inreg32xi8: +; CHECK: ## BB#0: +; CHECK-NEXT: vpbroadcastb %xmm0, %ymm0 +; CHECK-NEXT: retq %b = shufflevector <32 x i8> %a, <32 x i8> undef, <32 x i32> zeroinitializer ret <32 x i8> %b } -;CHECK-LABEL: _inreg16xi8: -;CHECK: vpbroadcastb -;CHECK: ret define <16 x i8> @_inreg16xi8(<16 x i8> %a) { +; CHECK-LABEL: _inreg16xi8: +; CHECK: ## BB#0: +; CHECK-NEXT: vpbroadcastb %xmm0, %xmm0 +; CHECK-NEXT: retq %b = shufflevector <16 x i8> %a, <16 x i8> undef, <16 x i32> zeroinitializer ret <16 x i8> %b } @@ -360,11 +629,11 @@ define <16 x i8> @_inreg16xi8(<16 x i8> %a) { ; formed from a concat_vectors (via the shufflevector) of two BUILD_VECTORs ; (via the insertelements). -; CHECK-LABEL: splat_concat1 -; CHECK-NOT: vinsertf128 -; CHECK: vbroadcastss -; CHECK-NEXT: ret define <8 x float> @splat_concat1(float %f) { +; CHECK-LABEL: splat_concat1: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastss %xmm0, %ymm0 +; CHECK-NEXT: retq %1 = insertelement <4 x float> undef, float %f, i32 0 %2 = insertelement <4 x float> %1, float %f, i32 1 %3 = insertelement <4 x float> %2, float %f, i32 2 @@ -373,11 +642,11 @@ define <8 x float> @splat_concat1(float %f) { ret <8 x float> %5 } -; CHECK-LABEL: splat_concat2 -; CHECK-NOT: vinsertf128 -; CHECK: vbroadcastss -; CHECK-NEXT: ret define <8 x float> @splat_concat2(float %f) { +; CHECK-LABEL: splat_concat2: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastss %xmm0, %ymm0 +; CHECK-NEXT: retq %1 = insertelement <4 x float> undef, float %f, i32 0 %2 = insertelement <4 x float> %1, float %f, i32 1 %3 = insertelement <4 x float> %2, float %f, i32 2 @@ -390,22 +659,22 @@ define <8 x float> @splat_concat2(float %f) { ret <8 x float> %9 } -; CHECK-LABEL: splat_concat3 -; CHECK-NOT: vinsertf128 -; CHECK: vbroadcastsd -; CHECK-NEXT: ret define <4 x double> @splat_concat3(double %d) { +; CHECK-LABEL: splat_concat3: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastsd %xmm0, %ymm0 +; CHECK-NEXT: retq %1 = insertelement <2 x double> undef, double %d, i32 0 %2 = insertelement <2 x double> %1, double %d, i32 1 %3 = shufflevector <2 x double> %2, <2 x double> undef, <4 x i32> ret <4 x double> %3 } -; CHECK-LABEL: splat_concat4 -; CHECK-NOT: vinsertf128 -; CHECK: vbroadcastsd -; CHECK-NEXT: ret define <4 x double> @splat_concat4(double %d) { +; CHECK-LABEL: splat_concat4: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastsd %xmm0, %ymm0 +; CHECK-NEXT: retq %1 = insertelement <2 x double> undef, double %d, i32 0 %2 = insertelement <2 x double> %1, double %d, i32 1 %3 = insertelement <2 x double> undef, double %d, i32 0 diff --git a/test/CodeGen/X86/avx512-arith.ll b/test/CodeGen/X86/avx512-arith.ll index 1ecd1007905a..9220e4f269cd 100644 --- a/test/CodeGen/X86/avx512-arith.ll +++ b/test/CodeGen/X86/avx512-arith.ll @@ -1,4 +1,9 @@ -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f | FileCheck --check-prefix=CHECK --check-prefix=AVX512F %s +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512vl | FileCheck --check-prefix=CHECK --check-prefix=AVX512VL %s +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512bw | FileCheck --check-prefix=CHECK --check-prefix=AVX512BW %s +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512dq | FileCheck --check-prefix=CHECK --check-prefix=AVX512DQ %s +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512dq -mattr=+avx512bw -mattr=+avx512vl| FileCheck --check-prefix=CHECK --check-prefix=SKX %s define <8 x double> @addpd512(<8 x double> %y, <8 x double> %x) { ; CHECK-LABEL: addpd512: @@ -83,18 +88,54 @@ entry: } define <8 x i64> @imulq512(<8 x i64> %y, <8 x i64> %x) { -; CHECK-LABEL: imulq512: -; CHECK: ## BB#0: -; CHECK-NEXT: vpmuludq %zmm0, %zmm1, %zmm2 -; CHECK-NEXT: vpsrlq $32, %zmm0, %zmm3 -; CHECK-NEXT: vpmuludq %zmm3, %zmm1, %zmm3 -; CHECK-NEXT: vpsllq $32, %zmm3, %zmm3 -; CHECK-NEXT: vpaddq %zmm3, %zmm2, %zmm2 -; CHECK-NEXT: vpsrlq $32, %zmm1, %zmm1 -; CHECK-NEXT: vpmuludq %zmm0, %zmm1, %zmm0 -; CHECK-NEXT: vpsllq $32, %zmm0, %zmm0 -; CHECK-NEXT: vpaddq %zmm0, %zmm2, %zmm0 -; CHECK-NEXT: retq +; AVX512F-LABEL: imulq512: +; AVX512F: ## BB#0: +; AVX512F-NEXT: vpmuludq %zmm0, %zmm1, %zmm2 +; AVX512F-NEXT: vpsrlq $32, %zmm0, %zmm3 +; AVX512F-NEXT: vpmuludq %zmm3, %zmm1, %zmm3 +; AVX512F-NEXT: vpsllq $32, %zmm3, %zmm3 +; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2 +; AVX512F-NEXT: vpsrlq $32, %zmm1, %zmm1 +; AVX512F-NEXT: vpmuludq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: vpsllq $32, %zmm0, %zmm0 +; AVX512F-NEXT: vpaddq %zmm0, %zmm2, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: imulq512: +; AVX512VL: ## BB#0: +; AVX512VL-NEXT: vpmuludq %zmm0, %zmm1, %zmm2 +; AVX512VL-NEXT: vpsrlq $32, %zmm0, %zmm3 +; AVX512VL-NEXT: vpmuludq %zmm3, %zmm1, %zmm3 +; AVX512VL-NEXT: vpsllq $32, %zmm3, %zmm3 +; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2 +; AVX512VL-NEXT: vpsrlq $32, %zmm1, %zmm1 +; AVX512VL-NEXT: vpmuludq %zmm0, %zmm1, %zmm0 +; AVX512VL-NEXT: vpsllq $32, %zmm0, %zmm0 +; AVX512VL-NEXT: vpaddq %zmm0, %zmm2, %zmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: imulq512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpmuludq %zmm0, %zmm1, %zmm2 +; AVX512BW-NEXT: vpsrlq $32, %zmm0, %zmm3 +; AVX512BW-NEXT: vpmuludq %zmm3, %zmm1, %zmm3 +; AVX512BW-NEXT: vpsllq $32, %zmm3, %zmm3 +; AVX512BW-NEXT: vpaddq %zmm3, %zmm2, %zmm2 +; AVX512BW-NEXT: vpsrlq $32, %zmm1, %zmm1 +; AVX512BW-NEXT: vpmuludq %zmm0, %zmm1, %zmm0 +; AVX512BW-NEXT: vpsllq $32, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddq %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512DQ-LABEL: imulq512: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpmullq %zmm0, %zmm1, %zmm0 +; AVX512DQ-NEXT: retq +; +; SKX-LABEL: imulq512: +; SKX: ## BB#0: +; SKX-NEXT: vpmullq %zmm0, %zmm1, %zmm0 +; SKX-NEXT: retq %z = mul <8 x i64>%x, %y ret <8 x i64>%z } @@ -463,10 +504,13 @@ entry: ret <8 x i64>%d } -; CHECK-LABEL: test_mask_vaddps -; CHECK: vaddps {{%zmm[0-9]{1,2}, %zmm[0-9]{1,2}, %zmm[0-9]{1,2} {%k[1-7]}}} -; CHECK: ret define <16 x float> @test_mask_vaddps(<16 x float> %dst, <16 x float> %i, +; CHECK-LABEL: test_mask_vaddps: +; CHECK: ## BB#0: +; CHECK-NEXT: vpxord %zmm4, %zmm4, %zmm4 +; CHECK-NEXT: vpcmpneqd %zmm4, %zmm3, %k1 +; CHECK-NEXT: vaddps %zmm2, %zmm1, %zmm0 {%k1} +; CHECK-NEXT: retq <16 x float> %j, <16 x i32> %mask1) nounwind readnone { %mask = icmp ne <16 x i32> %mask1, zeroinitializer @@ -475,10 +519,13 @@ define <16 x float> @test_mask_vaddps(<16 x float> %dst, <16 x float> %i, ret <16 x float> %r } -; CHECK-LABEL: test_mask_vmulps -; CHECK: vmulps {{%zmm[0-9]{1,2}, %zmm[0-9]{1,2}, %zmm[0-9]{1,2} {%k[1-7]}}} -; CHECK: ret define <16 x float> @test_mask_vmulps(<16 x float> %dst, <16 x float> %i, +; CHECK-LABEL: test_mask_vmulps: +; CHECK: ## BB#0: +; CHECK-NEXT: vpxord %zmm4, %zmm4, %zmm4 +; CHECK-NEXT: vpcmpneqd %zmm4, %zmm3, %k1 +; CHECK-NEXT: vmulps %zmm2, %zmm1, %zmm0 {%k1} +; CHECK-NEXT: retq <16 x float> %j, <16 x i32> %mask1) nounwind readnone { %mask = icmp ne <16 x i32> %mask1, zeroinitializer @@ -487,10 +534,13 @@ define <16 x float> @test_mask_vmulps(<16 x float> %dst, <16 x float> %i, ret <16 x float> %r } -; CHECK-LABEL: test_mask_vminps -; CHECK: vminps {{%zmm[0-9]{1,2}, %zmm[0-9]{1,2}, %zmm[0-9]{1,2} {%k[1-7]}}} -; CHECK: ret define <16 x float> @test_mask_vminps(<16 x float> %dst, <16 x float> %i, +; CHECK-LABEL: test_mask_vminps: +; CHECK: ## BB#0: +; CHECK-NEXT: vpxord %zmm4, %zmm4, %zmm4 +; CHECK-NEXT: vpcmpneqd %zmm4, %zmm3, %k1 +; CHECK-NEXT: vminps %zmm2, %zmm1, %zmm0 {%k1} +; CHECK-NEXT: retq <16 x float> %j, <16 x i32> %mask1) nounwind readnone { %mask = icmp ne <16 x i32> %mask1, zeroinitializer @@ -500,10 +550,41 @@ define <16 x float> @test_mask_vminps(<16 x float> %dst, <16 x float> %i, ret <16 x float> %r } -; CHECK-LABEL: test_mask_vminpd -; CHECK: vminpd {{%zmm[0-9]{1,2}, %zmm[0-9]{1,2}, %zmm[0-9]{1,2} {%k[1-7]}}} -; CHECK: ret define <8 x double> @test_mask_vminpd(<8 x double> %dst, <8 x double> %i, +; AVX512F-LABEL: test_mask_vminpd: +; AVX512F: ## BB#0: +; AVX512F-NEXT: vpxor %ymm4, %ymm4, %ymm4 +; AVX512F-NEXT: vpcmpneqd %zmm4, %zmm3, %k1 +; AVX512F-NEXT: vminpd %zmm2, %zmm1, %zmm0 {%k1} +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: test_mask_vminpd: +; AVX512VL: ## BB#0: +; AVX512VL-NEXT: vpxor %ymm4, %ymm4, %ymm4 +; AVX512VL-NEXT: vpcmpneqd %ymm4, %ymm3, %k1 +; AVX512VL-NEXT: vminpd %zmm2, %zmm1, %zmm0 {%k1} +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: test_mask_vminpd: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpxor %ymm4, %ymm4, %ymm4 +; AVX512BW-NEXT: vpcmpneqd %zmm4, %zmm3, %k1 +; AVX512BW-NEXT: vminpd %zmm2, %zmm1, %zmm0 {%k1} +; AVX512BW-NEXT: retq +; +; AVX512DQ-LABEL: test_mask_vminpd: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpxor %ymm4, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpcmpneqd %zmm4, %zmm3, %k1 +; AVX512DQ-NEXT: vminpd %zmm2, %zmm1, %zmm0 {%k1} +; AVX512DQ-NEXT: retq +; +; SKX-LABEL: test_mask_vminpd: +; SKX: ## BB#0: +; SKX-NEXT: vpxor %ymm4, %ymm4, %ymm4 +; SKX-NEXT: vpcmpneqd %ymm4, %ymm3, %k1 +; SKX-NEXT: vminpd %zmm2, %zmm1, %zmm0 {%k1} +; SKX-NEXT: retq <8 x double> %j, <8 x i32> %mask1) nounwind readnone { %mask = icmp ne <8 x i32> %mask1, zeroinitializer @@ -513,10 +594,13 @@ define <8 x double> @test_mask_vminpd(<8 x double> %dst, <8 x double> %i, ret <8 x double> %r } -; CHECK-LABEL: test_mask_vmaxps -; CHECK: vmaxps {{%zmm[0-9]{1,2}, %zmm[0-9]{1,2}, %zmm[0-9]{1,2} {%k[1-7]}}} -; CHECK: ret define <16 x float> @test_mask_vmaxps(<16 x float> %dst, <16 x float> %i, +; CHECK-LABEL: test_mask_vmaxps: +; CHECK: ## BB#0: +; CHECK-NEXT: vpxord %zmm4, %zmm4, %zmm4 +; CHECK-NEXT: vpcmpneqd %zmm4, %zmm3, %k1 +; CHECK-NEXT: vmaxps %zmm2, %zmm1, %zmm0 {%k1} +; CHECK-NEXT: retq <16 x float> %j, <16 x i32> %mask1) nounwind readnone { %mask = icmp ne <16 x i32> %mask1, zeroinitializer @@ -526,10 +610,41 @@ define <16 x float> @test_mask_vmaxps(<16 x float> %dst, <16 x float> %i, ret <16 x float> %r } -; CHECK-LABEL: test_mask_vmaxpd -; CHECK: vmaxpd {{%zmm[0-9]{1,2}, %zmm[0-9]{1,2}, %zmm[0-9]{1,2} {%k[1-7]}}} -; CHECK: ret define <8 x double> @test_mask_vmaxpd(<8 x double> %dst, <8 x double> %i, +; AVX512F-LABEL: test_mask_vmaxpd: +; AVX512F: ## BB#0: +; AVX512F-NEXT: vpxor %ymm4, %ymm4, %ymm4 +; AVX512F-NEXT: vpcmpneqd %zmm4, %zmm3, %k1 +; AVX512F-NEXT: vmaxpd %zmm2, %zmm1, %zmm0 {%k1} +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: test_mask_vmaxpd: +; AVX512VL: ## BB#0: +; AVX512VL-NEXT: vpxor %ymm4, %ymm4, %ymm4 +; AVX512VL-NEXT: vpcmpneqd %ymm4, %ymm3, %k1 +; AVX512VL-NEXT: vmaxpd %zmm2, %zmm1, %zmm0 {%k1} +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: test_mask_vmaxpd: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpxor %ymm4, %ymm4, %ymm4 +; AVX512BW-NEXT: vpcmpneqd %zmm4, %zmm3, %k1 +; AVX512BW-NEXT: vmaxpd %zmm2, %zmm1, %zmm0 {%k1} +; AVX512BW-NEXT: retq +; +; AVX512DQ-LABEL: test_mask_vmaxpd: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpxor %ymm4, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpcmpneqd %zmm4, %zmm3, %k1 +; AVX512DQ-NEXT: vmaxpd %zmm2, %zmm1, %zmm0 {%k1} +; AVX512DQ-NEXT: retq +; +; SKX-LABEL: test_mask_vmaxpd: +; SKX: ## BB#0: +; SKX-NEXT: vpxor %ymm4, %ymm4, %ymm4 +; SKX-NEXT: vpcmpneqd %ymm4, %ymm3, %k1 +; SKX-NEXT: vmaxpd %zmm2, %zmm1, %zmm0 {%k1} +; SKX-NEXT: retq <8 x double> %j, <8 x i32> %mask1) nounwind readnone { %mask = icmp ne <8 x i32> %mask1, zeroinitializer @@ -539,10 +654,13 @@ define <8 x double> @test_mask_vmaxpd(<8 x double> %dst, <8 x double> %i, ret <8 x double> %r } -; CHECK-LABEL: test_mask_vsubps -; CHECK: vsubps {{%zmm[0-9]{1,2}, %zmm[0-9]{1,2}, %zmm[0-9]{1,2} {%k[1-7]}}} -; CHECK: ret define <16 x float> @test_mask_vsubps(<16 x float> %dst, <16 x float> %i, +; CHECK-LABEL: test_mask_vsubps: +; CHECK: ## BB#0: +; CHECK-NEXT: vpxord %zmm4, %zmm4, %zmm4 +; CHECK-NEXT: vpcmpneqd %zmm4, %zmm3, %k1 +; CHECK-NEXT: vsubps %zmm2, %zmm1, %zmm0 {%k1} +; CHECK-NEXT: retq <16 x float> %j, <16 x i32> %mask1) nounwind readnone { %mask = icmp ne <16 x i32> %mask1, zeroinitializer @@ -551,10 +669,13 @@ define <16 x float> @test_mask_vsubps(<16 x float> %dst, <16 x float> %i, ret <16 x float> %r } -; CHECK-LABEL: test_mask_vdivps -; CHECK: vdivps {{%zmm[0-9]{1,2}, %zmm[0-9]{1,2}, %zmm[0-9]{1,2} {%k[1-7]}}} -; CHECK: ret define <16 x float> @test_mask_vdivps(<16 x float> %dst, <16 x float> %i, +; CHECK-LABEL: test_mask_vdivps: +; CHECK: ## BB#0: +; CHECK-NEXT: vpxord %zmm4, %zmm4, %zmm4 +; CHECK-NEXT: vpcmpneqd %zmm4, %zmm3, %k1 +; CHECK-NEXT: vdivps %zmm2, %zmm1, %zmm0 {%k1} +; CHECK-NEXT: retq <16 x float> %j, <16 x i32> %mask1) nounwind readnone { %mask = icmp ne <16 x i32> %mask1, zeroinitializer @@ -563,10 +684,13 @@ define <16 x float> @test_mask_vdivps(<16 x float> %dst, <16 x float> %i, ret <16 x float> %r } -; CHECK-LABEL: test_mask_vaddpd -; CHECK: vaddpd {{%zmm[0-9]{1,2}, %zmm[0-9]{1,2}, %zmm[0-9]{1,2} {%k[1-7]}}} -; CHECK: ret define <8 x double> @test_mask_vaddpd(<8 x double> %dst, <8 x double> %i, +; CHECK-LABEL: test_mask_vaddpd: +; CHECK: ## BB#0: +; CHECK-NEXT: vpxord %zmm4, %zmm4, %zmm4 +; CHECK-NEXT: vpcmpneqq %zmm4, %zmm3, %k1 +; CHECK-NEXT: vaddpd %zmm2, %zmm1, %zmm0 {%k1} +; CHECK-NEXT: retq <8 x double> %j, <8 x i64> %mask1) nounwind readnone { %mask = icmp ne <8 x i64> %mask1, zeroinitializer @@ -575,10 +699,13 @@ define <8 x double> @test_mask_vaddpd(<8 x double> %dst, <8 x double> %i, ret <8 x double> %r } -; CHECK-LABEL: test_maskz_vaddpd -; CHECK: vaddpd {{%zmm[0-9]{1,2}, %zmm[0-9]{1,2}, %zmm[0-9]{1,2} {%k[1-7]} {z}}} -; CHECK: ret define <8 x double> @test_maskz_vaddpd(<8 x double> %i, <8 x double> %j, +; CHECK-LABEL: test_maskz_vaddpd: +; CHECK: ## BB#0: +; CHECK-NEXT: vpxord %zmm3, %zmm3, %zmm3 +; CHECK-NEXT: vpcmpneqq %zmm3, %zmm2, %k1 +; CHECK-NEXT: vaddpd %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq <8 x i64> %mask1) nounwind readnone { %mask = icmp ne <8 x i64> %mask1, zeroinitializer %x = fadd <8 x double> %i, %j @@ -586,10 +713,13 @@ define <8 x double> @test_maskz_vaddpd(<8 x double> %i, <8 x double> %j, ret <8 x double> %r } -; CHECK-LABEL: test_mask_fold_vaddpd -; CHECK: vaddpd (%rdi), {{.*%zmm[0-9]{1,2}, %zmm[0-9]{1,2} {%k[1-7]}.*}} -; CHECK: ret define <8 x double> @test_mask_fold_vaddpd(<8 x double> %dst, <8 x double> %i, +; CHECK-LABEL: test_mask_fold_vaddpd: +; CHECK: ## BB#0: +; CHECK-NEXT: vpxord %zmm3, %zmm3, %zmm3 +; CHECK-NEXT: vpcmpneqq %zmm3, %zmm2, %k1 +; CHECK-NEXT: vaddpd (%rdi), %zmm1, %zmm0 {%k1} +; CHECK-NEXT: retq <8 x double>* %j, <8 x i64> %mask1) nounwind { %mask = icmp ne <8 x i64> %mask1, zeroinitializer @@ -599,10 +729,13 @@ define <8 x double> @test_mask_fold_vaddpd(<8 x double> %dst, <8 x double> %i, ret <8 x double> %r } -; CHECK-LABEL: test_maskz_fold_vaddpd -; CHECK: vaddpd (%rdi), {{.*%zmm[0-9]{1,2}, %zmm[0-9]{1,2} {%k[1-7]} {z}.*}} -; CHECK: ret define <8 x double> @test_maskz_fold_vaddpd(<8 x double> %i, <8 x double>* %j, +; CHECK-LABEL: test_maskz_fold_vaddpd: +; CHECK: ## BB#0: +; CHECK-NEXT: vpxord %zmm2, %zmm2, %zmm2 +; CHECK-NEXT: vpcmpneqq %zmm2, %zmm1, %k1 +; CHECK-NEXT: vaddpd (%rdi), %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq <8 x i64> %mask1) nounwind { %mask = icmp ne <8 x i64> %mask1, zeroinitializer %tmp = load <8 x double>, <8 x double>* %j, align 8 @@ -611,10 +744,11 @@ define <8 x double> @test_maskz_fold_vaddpd(<8 x double> %i, <8 x double>* %j, ret <8 x double> %r } -; CHECK-LABEL: test_broadcast_vaddpd -; CHECK: vaddpd (%rdi){1to8}, %zmm{{.*}} -; CHECK: ret define <8 x double> @test_broadcast_vaddpd(<8 x double> %i, double* %j) nounwind { +; CHECK-LABEL: test_broadcast_vaddpd: +; CHECK: ## BB#0: +; CHECK-NEXT: vaddpd (%rdi){1to8}, %zmm0, %zmm0 +; CHECK-NEXT: retq %tmp = load double, double* %j %b = insertelement <8 x double> undef, double %tmp, i32 0 %c = shufflevector <8 x double> %b, <8 x double> undef, @@ -623,10 +757,14 @@ define <8 x double> @test_broadcast_vaddpd(<8 x double> %i, double* %j) nounwind ret <8 x double> %x } -; CHECK-LABEL: test_mask_broadcast_vaddpd -; CHECK: vaddpd (%rdi){1to8}, %zmm{{.*{%k[1-7]}.*}} -; CHECK: ret define <8 x double> @test_mask_broadcast_vaddpd(<8 x double> %dst, <8 x double> %i, +; CHECK-LABEL: test_mask_broadcast_vaddpd: +; CHECK: ## BB#0: +; CHECK-NEXT: vpxord %zmm0, %zmm0, %zmm0 +; CHECK-NEXT: vpcmpneqq %zmm0, %zmm2, %k1 +; CHECK-NEXT: vaddpd (%rdi){1to8}, %zmm1, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq double* %j, <8 x i64> %mask1) nounwind { %mask = icmp ne <8 x i64> %mask1, zeroinitializer %tmp = load double, double* %j @@ -638,10 +776,13 @@ define <8 x double> @test_mask_broadcast_vaddpd(<8 x double> %dst, <8 x double> ret <8 x double> %r } -; CHECK-LABEL: test_maskz_broadcast_vaddpd -; CHECK: vaddpd (%rdi){1to8}, %zmm{{.*{%k[1-7]} {z}.*}} -; CHECK: ret define <8 x double> @test_maskz_broadcast_vaddpd(<8 x double> %i, double* %j, +; CHECK-LABEL: test_maskz_broadcast_vaddpd: +; CHECK: ## BB#0: +; CHECK-NEXT: vpxord %zmm2, %zmm2, %zmm2 +; CHECK-NEXT: vpcmpneqq %zmm2, %zmm1, %k1 +; CHECK-NEXT: vaddpd (%rdi){1to8}, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq <8 x i64> %mask1) nounwind { %mask = icmp ne <8 x i64> %mask1, zeroinitializer %tmp = load double, double* %j @@ -652,3 +793,104 @@ define <8 x double> @test_maskz_broadcast_vaddpd(<8 x double> %i, double* %j, %r = select <8 x i1> %mask, <8 x double> %x, <8 x double> zeroinitializer ret <8 x double> %r } + +define <16 x float> @test_fxor(<16 x float> %a) { +; AVX512F-LABEL: test_fxor: +; AVX512F: ## BB#0: +; AVX512F-NEXT: vpxord {{.*}}(%rip), %zmm0, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: test_fxor: +; AVX512VL: ## BB#0: +; AVX512VL-NEXT: vpxord {{.*}}(%rip), %zmm0, %zmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: test_fxor: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpxord {{.*}}(%rip), %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512DQ-LABEL: test_fxor: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vxorps {{.*}}(%rip), %zmm0, %zmm0 +; AVX512DQ-NEXT: retq +; +; SKX-LABEL: test_fxor: +; SKX: ## BB#0: +; SKX-NEXT: vxorps {{.*}}(%rip), %zmm0, %zmm0 +; SKX-NEXT: retq + + %res = fsub <16 x float> , %a + ret <16 x float>%res +} + +define <8 x float> @test_fxor_8f32(<8 x float> %a) { +; CHECK-LABEL: test_fxor_8f32: +; CHECK: ## BB#0: +; CHECK-NEXT: vxorps {{.*}}(%rip), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = fsub <8 x float> , %a + ret <8 x float>%res +} + +define <8 x double> @fabs_v8f64(<8 x double> %p) +; AVX512F-LABEL: fabs_v8f64: +; AVX512F: ## BB#0: +; AVX512F-NEXT: vpandq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: fabs_v8f64: +; AVX512VL: ## BB#0: +; AVX512VL-NEXT: vpandq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: fabs_v8f64: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpandq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512DQ-LABEL: fabs_v8f64: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vandpd {{.*}}(%rip), %zmm0, %zmm0 +; AVX512DQ-NEXT: retq +; +; SKX-LABEL: fabs_v8f64: +; SKX: ## BB#0: +; SKX-NEXT: vandpd {{.*}}(%rip), %zmm0, %zmm0 +; SKX-NEXT: retq +{ + %t = call <8 x double> @llvm.fabs.v8f64(<8 x double> %p) + ret <8 x double> %t +} +declare <8 x double> @llvm.fabs.v8f64(<8 x double> %p) + +define <16 x float> @fabs_v16f32(<16 x float> %p) +; AVX512F-LABEL: fabs_v16f32: +; AVX512F: ## BB#0: +; AVX512F-NEXT: vpandd {{.*}}(%rip), %zmm0, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: fabs_v16f32: +; AVX512VL: ## BB#0: +; AVX512VL-NEXT: vpandd {{.*}}(%rip), %zmm0, %zmm0 +; AVX512VL-NEXT: retq +; +; AVX512BW-LABEL: fabs_v16f32: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpandd {{.*}}(%rip), %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512DQ-LABEL: fabs_v16f32: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vandps {{.*}}(%rip), %zmm0, %zmm0 +; AVX512DQ-NEXT: retq +; +; SKX-LABEL: fabs_v16f32: +; SKX: ## BB#0: +; SKX-NEXT: vandps {{.*}}(%rip), %zmm0, %zmm0 +; SKX-NEXT: retq +{ + %t = call <16 x float> @llvm.fabs.v16f32(<16 x float> %p) + ret <16 x float> %t +} +declare <16 x float> @llvm.fabs.v16f32(<16 x float> %p) diff --git a/test/CodeGen/X86/avx512-bugfix-25270.ll b/test/CodeGen/X86/avx512-bugfix-25270.ll new file mode 100644 index 000000000000..d024475274b4 --- /dev/null +++ b/test/CodeGen/X86/avx512-bugfix-25270.ll @@ -0,0 +1,35 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s + +declare void @Print__512(<16 x i32>) #0 + +define void @bar__512(<16 x i32>* %var) #0 { +; CHECK-LABEL: bar__512: +; CHECK: ## BB#0: ## %allocas +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: subq $112, %rsp +; CHECK-NEXT: movq %rdi, %rbx +; CHECK-NEXT: vmovdqu32 (%rbx), %zmm0 +; CHECK-NEXT: vmovups %zmm0, (%rsp) ## 64-byte Spill +; CHECK-NEXT: vpbroadcastd {{.*}}(%rip), %zmm1 +; CHECK-NEXT: vmovdqa32 %zmm1, (%rbx) +; CHECK-NEXT: callq _Print__512 +; CHECK-NEXT: vmovups (%rsp), %zmm0 ## 64-byte Reload +; CHECK-NEXT: callq _Print__512 +; CHECK-NEXT: vpbroadcastd {{.*}}(%rip), %zmm0 +; CHECK-NEXT: vmovdqa32 %zmm0, (%rbx) +; CHECK-NEXT: addq $112, %rsp +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: retq +allocas: + %var_load_load = load <16 x i32>, <16 x i32>* %var, align 1 + store <16 x i32> , <16 x i32>* %var, align 64 + call void @Print__512(<16 x i32> %var_load_load) + ; %var_load_load value should be reloaded + call void @Print__512(<16 x i32> %var_load_load) + store <16 x i32> , <16 x i32>* %var, align 64 + ret void +} + + +attributes #0 = { nounwind } diff --git a/test/CodeGen/X86/avx512-build-vector.ll b/test/CodeGen/X86/avx512-build-vector.ll index e5373c575c1a..0f89aa71162e 100644 --- a/test/CodeGen/X86/avx512-build-vector.ll +++ b/test/CodeGen/X86/avx512-build-vector.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -march=x86-64 -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s define <16 x i32> @test2(<16 x i32> %x) { diff --git a/test/CodeGen/X86/avx512-calling-conv.ll b/test/CodeGen/X86/avx512-calling-conv.ll index edb6bef1a4ac..a61aeba5aff9 100644 --- a/test/CodeGen/X86/avx512-calling-conv.ll +++ b/test/CodeGen/X86/avx512-calling-conv.ll @@ -1,55 +1,167 @@ -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s --check-prefix=KNL -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s --check-prefix=SKX +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s --check-prefix=ALL_X64 --check-prefix=KNL +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s --check-prefix=ALL_X64 --check-prefix=SKX ; RUN: llc < %s -mtriple=i686-apple-darwin -mcpu=knl | FileCheck %s --check-prefix=KNL_X32 -; KNL-LABEL: test1 -; KNL: vxorps define <16 x i1> @test1() { +; ALL_X64-LABEL: test1: +; ALL_X64: ## BB#0: +; ALL_X64-NEXT: vxorps %xmm0, %xmm0, %xmm0 +; ALL_X64-NEXT: retq +; +; KNL_X32-LABEL: test1: +; KNL_X32: ## BB#0: +; KNL_X32-NEXT: vxorps %xmm0, %xmm0, %xmm0 +; KNL_X32-NEXT: retl ret <16 x i1> zeroinitializer } -; SKX-LABEL: test2 -; SKX: vpmovb2m -; SKX: vpmovb2m -; SKX: kandw -; SKX: vpmovm2b -; KNL-LABEL: test2 -; KNL: vpmovsxbd -; KNL: vpmovsxbd -; KNL: vpandd -; KNL: vpmovdb define <16 x i1> @test2(<16 x i1>%a, <16 x i1>%b) { +; KNL-LABEL: test2: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxbd %xmm1, %zmm1 +; KNL-NEXT: vpslld $31, %zmm1, %zmm1 +; KNL-NEXT: vpmovsxbd %xmm0, %zmm0 +; KNL-NEXT: vpslld $31, %zmm0, %zmm0 +; KNL-NEXT: vptestmd %zmm0, %zmm0, %k1 +; KNL-NEXT: vptestmd %zmm1, %zmm1, %k1 {%k1} +; KNL-NEXT: vpbroadcastd {{.*}}(%rip), %zmm0 {%k1} {z} +; KNL-NEXT: vpmovdb %zmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test2: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm1, %xmm1 +; SKX-NEXT: vpmovb2m %xmm1, %k0 +; SKX-NEXT: vpsllw $7, %xmm0, %xmm0 +; SKX-NEXT: vpmovb2m %xmm0, %k1 +; SKX-NEXT: kandw %k0, %k1, %k0 +; SKX-NEXT: vpmovm2b %k0, %xmm0 +; SKX-NEXT: retq +; +; KNL_X32-LABEL: test2: +; KNL_X32: ## BB#0: +; KNL_X32-NEXT: vpmovsxbd %xmm1, %zmm1 +; KNL_X32-NEXT: vpslld $31, %zmm1, %zmm1 +; KNL_X32-NEXT: vpmovsxbd %xmm0, %zmm0 +; KNL_X32-NEXT: vpslld $31, %zmm0, %zmm0 +; KNL_X32-NEXT: vptestmd %zmm0, %zmm0, %k1 +; KNL_X32-NEXT: vptestmd %zmm1, %zmm1, %k1 {%k1} +; KNL_X32-NEXT: vpbroadcastd LCPI1_0, %zmm0 {%k1} {z} +; KNL_X32-NEXT: vpmovdb %zmm0, %xmm0 +; KNL_X32-NEXT: retl %c = and <16 x i1>%a, %b ret <16 x i1> %c } -; SKX-LABEL: test3 -; SKX: vpmovw2m -; SKX: vpmovw2m -; SKX: kandb -; SKX: vpmovm2w define <8 x i1> @test3(<8 x i1>%a, <8 x i1>%b) { +; KNL-LABEL: test3: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwq %xmm1, %zmm1 +; KNL-NEXT: vpsllq $63, %zmm1, %zmm1 +; KNL-NEXT: vpmovsxwq %xmm0, %zmm0 +; KNL-NEXT: vpsllq $63, %zmm0, %zmm0 +; KNL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; KNL-NEXT: vptestmq %zmm1, %zmm1, %k1 {%k1} +; KNL-NEXT: vpbroadcastq {{.*}}(%rip), %zmm0 {%k1} {z} +; KNL-NEXT: vpmovqw %zmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test3: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $15, %xmm1, %xmm1 +; SKX-NEXT: vpmovw2m %xmm1, %k0 +; SKX-NEXT: vpsllw $15, %xmm0, %xmm0 +; SKX-NEXT: vpmovw2m %xmm0, %k1 +; SKX-NEXT: kandb %k0, %k1, %k0 +; SKX-NEXT: vpmovm2w %k0, %xmm0 +; SKX-NEXT: retq +; +; KNL_X32-LABEL: test3: +; KNL_X32: ## BB#0: +; KNL_X32-NEXT: vpmovsxwq %xmm1, %zmm1 +; KNL_X32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [63,0,63,0,63,0,63,0,63,0,63,0,63,0,63,0] +; KNL_X32-NEXT: vpsllvq %zmm2, %zmm1, %zmm1 +; KNL_X32-NEXT: vpmovsxwq %xmm0, %zmm0 +; KNL_X32-NEXT: vpsllvq %zmm2, %zmm0, %zmm0 +; KNL_X32-NEXT: vptestmq %zmm0, %zmm0, %k1 +; KNL_X32-NEXT: vptestmq %zmm1, %zmm1, %k1 {%k1} +; KNL_X32-NEXT: vpbroadcastd LCPI2_1, %zmm0 +; KNL_X32-NEXT: vmovdqu64 %zmm0, %zmm0 {%k1} {z} +; KNL_X32-NEXT: vpmovqw %zmm0, %xmm0 +; KNL_X32-NEXT: retl %c = and <8 x i1>%a, %b ret <8 x i1> %c } -; SKX-LABEL: test4 -; SKX: vpmovd2m -; SKX: vpmovd2m -; SKX: kandw -; SKX: vpmovm2d define <4 x i1> @test4(<4 x i1>%a, <4 x i1>%b) { +; KNL-LABEL: test4: +; KNL: ## BB#0: +; KNL-NEXT: vandps %xmm1, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test4: +; SKX: ## BB#0: +; SKX-NEXT: vpslld $31, %xmm0, %xmm0 +; SKX-NEXT: vpmovd2m %xmm0, %k0 +; SKX-NEXT: vpslld $31, %xmm1, %xmm0 +; SKX-NEXT: vpmovd2m %xmm0, %k1 +; SKX-NEXT: kandw %k1, %k0, %k0 +; SKX-NEXT: vpmovm2d %k0, %xmm0 +; SKX-NEXT: retq +; +; KNL_X32-LABEL: test4: +; KNL_X32: ## BB#0: +; KNL_X32-NEXT: vandps %xmm1, %xmm0, %xmm0 +; KNL_X32-NEXT: retl %c = and <4 x i1>%a, %b ret <4 x i1> %c } -; SKX-LABEL: test5 -; SKX: vpcmpgtd -; SKX: vpmovm2w -; SKX: call -; SKX: vpmovzxwd declare <8 x i1> @func8xi1(<8 x i1> %a) + define <8 x i32> @test5(<8 x i32>%a, <8 x i32>%b) { +; KNL-LABEL: test5: +; KNL: ## BB#0: +; KNL-NEXT: pushq %rax +; KNL-NEXT: Ltmp0: +; KNL-NEXT: .cfi_def_cfa_offset 16 +; KNL-NEXT: vpcmpgtd %ymm1, %ymm0, %ymm0 +; KNL-NEXT: vpmovdw %zmm0, %ymm0 +; KNL-NEXT: callq _func8xi1 +; KNL-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; KNL-NEXT: vpslld $31, %ymm0, %ymm0 +; KNL-NEXT: vpsrad $31, %ymm0, %ymm0 +; KNL-NEXT: popq %rax +; KNL-NEXT: retq +; +; SKX-LABEL: test5: +; SKX: ## BB#0: +; SKX-NEXT: pushq %rax +; SKX-NEXT: Ltmp0: +; SKX-NEXT: .cfi_def_cfa_offset 16 +; SKX-NEXT: vpcmpgtd %ymm1, %ymm0, %k0 +; SKX-NEXT: vpmovm2w %k0, %xmm0 +; SKX-NEXT: callq _func8xi1 +; SKX-NEXT: vpmovzxwd %xmm0, %ymm0 +; SKX-NEXT: vpslld $31, %ymm0, %ymm0 +; SKX-NEXT: vpsrad $31, %ymm0, %ymm0 +; SKX-NEXT: popq %rax +; SKX-NEXT: retq +; +; KNL_X32-LABEL: test5: +; KNL_X32: ## BB#0: +; KNL_X32-NEXT: subl $12, %esp +; KNL_X32-NEXT: Ltmp0: +; KNL_X32-NEXT: .cfi_def_cfa_offset 16 +; KNL_X32-NEXT: vpcmpgtd %ymm1, %ymm0, %ymm0 +; KNL_X32-NEXT: vpmovdw %zmm0, %ymm0 +; KNL_X32-NEXT: calll L_func8xi1$stub +; KNL_X32-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; KNL_X32-NEXT: vpslld $31, %ymm0, %ymm0 +; KNL_X32-NEXT: vpsrad $31, %ymm0, %ymm0 +; KNL_X32-NEXT: addl $12, %esp +; KNL_X32-NEXT: retl %cmpRes = icmp sgt <8 x i32>%a, %b %resi = call <8 x i1> @func8xi1(<8 x i1> %cmpRes) %res = sext <8 x i1>%resi to <8 x i32> @@ -58,14 +170,50 @@ define <8 x i32> @test5(<8 x i32>%a, <8 x i32>%b) { declare <16 x i1> @func16xi1(<16 x i1> %a) -; KNL-LABEL: test6 -; KNL: vpbroadcastd -; KNL: vpmovdb -; KNL: call -; KNL: vpmovzxbd -; KNL: vpslld $31, %zmm -; KNL: vpsrad $31, %zmm define <16 x i32> @test6(<16 x i32>%a, <16 x i32>%b) { +; KNL-LABEL: test6: +; KNL: ## BB#0: +; KNL-NEXT: pushq %rax +; KNL-NEXT: Ltmp1: +; KNL-NEXT: .cfi_def_cfa_offset 16 +; KNL-NEXT: vpcmpgtd %zmm1, %zmm0, %k1 +; KNL-NEXT: vpbroadcastd {{.*}}(%rip), %zmm0 {%k1} {z} +; KNL-NEXT: vpmovdb %zmm0, %xmm0 +; KNL-NEXT: callq _func16xi1 +; KNL-NEXT: vpmovzxbd %xmm0, %zmm0 +; KNL-NEXT: vpslld $31, %zmm0, %zmm0 +; KNL-NEXT: vpsrad $31, %zmm0, %zmm0 +; KNL-NEXT: popq %rax +; KNL-NEXT: retq +; +; SKX-LABEL: test6: +; SKX: ## BB#0: +; SKX-NEXT: pushq %rax +; SKX-NEXT: Ltmp1: +; SKX-NEXT: .cfi_def_cfa_offset 16 +; SKX-NEXT: vpcmpgtd %zmm1, %zmm0, %k0 +; SKX-NEXT: vpmovm2b %k0, %xmm0 +; SKX-NEXT: callq _func16xi1 +; SKX-NEXT: vpmovzxbd %xmm0, %zmm0 +; SKX-NEXT: vpslld $31, %zmm0, %zmm0 +; SKX-NEXT: vpsrad $31, %zmm0, %zmm0 +; SKX-NEXT: popq %rax +; SKX-NEXT: retq +; +; KNL_X32-LABEL: test6: +; KNL_X32: ## BB#0: +; KNL_X32-NEXT: subl $12, %esp +; KNL_X32-NEXT: Ltmp1: +; KNL_X32-NEXT: .cfi_def_cfa_offset 16 +; KNL_X32-NEXT: vpcmpgtd %zmm1, %zmm0, %k1 +; KNL_X32-NEXT: vpbroadcastd LCPI5_0, %zmm0 {%k1} {z} +; KNL_X32-NEXT: vpmovdb %zmm0, %xmm0 +; KNL_X32-NEXT: calll L_func16xi1$stub +; KNL_X32-NEXT: vpmovzxbd %xmm0, %zmm0 +; KNL_X32-NEXT: vpslld $31, %zmm0, %zmm0 +; KNL_X32-NEXT: vpsrad $31, %zmm0, %zmm0 +; KNL_X32-NEXT: addl $12, %esp +; KNL_X32-NEXT: retl %cmpRes = icmp sgt <16 x i32>%a, %b %resi = call <16 x i1> @func16xi1(<16 x i1> %cmpRes) %res = sext <16 x i1>%resi to <16 x i32> @@ -73,82 +221,265 @@ define <16 x i32> @test6(<16 x i32>%a, <16 x i32>%b) { } declare <4 x i1> @func4xi1(<4 x i1> %a) -; SKX-LABEL: test7 -; SKX: vpmovm2d -; SKX: call -; SKX: vpslld $31, %xmm -; SKX: vpsrad $31, %xmm define <4 x i32> @test7(<4 x i32>%a, <4 x i32>%b) { +; KNL-LABEL: test7: +; KNL: ## BB#0: +; KNL-NEXT: pushq %rax +; KNL-NEXT: Ltmp2: +; KNL-NEXT: .cfi_def_cfa_offset 16 +; KNL-NEXT: vpcmpgtd %xmm1, %xmm0, %xmm0 +; KNL-NEXT: callq _func4xi1 +; KNL-NEXT: vpslld $31, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: popq %rax +; KNL-NEXT: retq +; +; SKX-LABEL: test7: +; SKX: ## BB#0: +; SKX-NEXT: pushq %rax +; SKX-NEXT: Ltmp2: +; SKX-NEXT: .cfi_def_cfa_offset 16 +; SKX-NEXT: vpcmpgtd %xmm1, %xmm0, %k0 +; SKX-NEXT: vpmovm2d %k0, %xmm0 +; SKX-NEXT: callq _func4xi1 +; SKX-NEXT: vpslld $31, %xmm0, %xmm0 +; SKX-NEXT: vpsrad $31, %xmm0, %xmm0 +; SKX-NEXT: popq %rax +; SKX-NEXT: retq +; +; KNL_X32-LABEL: test7: +; KNL_X32: ## BB#0: +; KNL_X32-NEXT: subl $12, %esp +; KNL_X32-NEXT: Ltmp2: +; KNL_X32-NEXT: .cfi_def_cfa_offset 16 +; KNL_X32-NEXT: vpcmpgtd %xmm1, %xmm0, %xmm0 +; KNL_X32-NEXT: calll L_func4xi1$stub +; KNL_X32-NEXT: vpslld $31, %xmm0, %xmm0 +; KNL_X32-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL_X32-NEXT: addl $12, %esp +; KNL_X32-NEXT: retl %cmpRes = icmp sgt <4 x i32>%a, %b %resi = call <4 x i1> @func4xi1(<4 x i1> %cmpRes) %res = sext <4 x i1>%resi to <4 x i32> ret <4 x i32> %res } -; SKX-LABEL: test7a -; SKX: call -; SKX: vpmovw2m %xmm0, %k0 -; SKX: kandb define <8 x i1> @test7a(<8 x i32>%a, <8 x i32>%b) { +; KNL-LABEL: test7a: +; KNL: ## BB#0: +; KNL-NEXT: pushq %rax +; KNL-NEXT: Ltmp3: +; KNL-NEXT: .cfi_def_cfa_offset 16 +; KNL-NEXT: vpcmpgtd %ymm1, %ymm0, %ymm0 +; KNL-NEXT: vpmovdw %zmm0, %ymm0 +; KNL-NEXT: callq _func8xi1 +; KNL-NEXT: vpmovsxwq %xmm0, %zmm0 +; KNL-NEXT: vpsllq $63, %zmm0, %zmm0 +; KNL-NEXT: movb $85, %al +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: kmovw %eax, %k1 +; KNL-NEXT: vptestmq %zmm0, %zmm0, %k1 {%k1} +; KNL-NEXT: vpbroadcastq {{.*}}(%rip), %zmm0 {%k1} {z} +; KNL-NEXT: vpmovqw %zmm0, %xmm0 +; KNL-NEXT: popq %rax +; KNL-NEXT: retq +; +; SKX-LABEL: test7a: +; SKX: ## BB#0: +; SKX-NEXT: pushq %rax +; SKX-NEXT: Ltmp3: +; SKX-NEXT: .cfi_def_cfa_offset 16 +; SKX-NEXT: vpcmpgtd %ymm1, %ymm0, %k0 +; SKX-NEXT: vpmovm2w %k0, %xmm0 +; SKX-NEXT: callq _func8xi1 +; SKX-NEXT: vpsllw $15, %xmm0, %xmm0 +; SKX-NEXT: vpmovw2m %xmm0, %k0 +; SKX-NEXT: movb $85, %al +; SKX-NEXT: kmovb %eax, %k1 +; SKX-NEXT: kandb %k1, %k0, %k0 +; SKX-NEXT: vpmovm2w %k0, %xmm0 +; SKX-NEXT: popq %rax +; SKX-NEXT: retq +; +; KNL_X32-LABEL: test7a: +; KNL_X32: ## BB#0: +; KNL_X32-NEXT: subl $12, %esp +; KNL_X32-NEXT: Ltmp3: +; KNL_X32-NEXT: .cfi_def_cfa_offset 16 +; KNL_X32-NEXT: vpcmpgtd %ymm1, %ymm0, %ymm0 +; KNL_X32-NEXT: vpmovdw %zmm0, %ymm0 +; KNL_X32-NEXT: calll L_func8xi1$stub +; KNL_X32-NEXT: vpmovsxwq %xmm0, %zmm0 +; KNL_X32-NEXT: vpsllvq LCPI7_0, %zmm0, %zmm0 +; KNL_X32-NEXT: movb $85, %al +; KNL_X32-NEXT: movzbl %al, %eax +; KNL_X32-NEXT: kmovw %eax, %k1 +; KNL_X32-NEXT: vptestmq %zmm0, %zmm0, %k1 {%k1} +; KNL_X32-NEXT: vpbroadcastd LCPI7_1, %zmm0 +; KNL_X32-NEXT: vmovdqu64 %zmm0, %zmm0 {%k1} {z} +; KNL_X32-NEXT: vpmovqw %zmm0, %xmm0 +; KNL_X32-NEXT: addl $12, %esp +; KNL_X32-NEXT: retl %cmpRes = icmp sgt <8 x i32>%a, %b %resi = call <8 x i1> @func8xi1(<8 x i1> %cmpRes) %res = and <8 x i1>%resi, ret <8 x i1> %res } - -; KNL_X32-LABEL: test8 -; KNL_X32: testb $1, 4(%esp) -; KNL_X32:jne - -; KNL-LABEL: test8 -; KNL: testb $1, %dil -; KNL:jne - define <16 x i8> @test8(<16 x i8> %a1, <16 x i8> %a2, i1 %cond) { +; ALL_X64-LABEL: test8: +; ALL_X64: ## BB#0: +; ALL_X64-NEXT: testb $1, %dil +; ALL_X64-NEXT: jne LBB8_2 +; ALL_X64-NEXT: ## BB#1: +; ALL_X64-NEXT: vmovaps %zmm1, %zmm0 +; ALL_X64-NEXT: LBB8_2: +; ALL_X64-NEXT: retq +; +; KNL_X32-LABEL: test8: +; KNL_X32: ## BB#0: +; KNL_X32-NEXT: testb $1, {{[0-9]+}}(%esp) +; KNL_X32-NEXT: jne LBB8_2 +; KNL_X32-NEXT: ## BB#1: +; KNL_X32-NEXT: vmovaps %zmm1, %zmm0 +; KNL_X32-NEXT: LBB8_2: +; KNL_X32-NEXT: retl %res = select i1 %cond, <16 x i8> %a1, <16 x i8> %a2 ret <16 x i8> %res } -; KNL-LABEL: test9 -; KNL: vucomisd -; KNL: setb define i1 @test9(double %a, double %b) { +; ALL_X64-LABEL: test9: +; ALL_X64: ## BB#0: +; ALL_X64-NEXT: vucomisd %xmm0, %xmm1 +; ALL_X64-NEXT: setb %al +; ALL_X64-NEXT: retq +; +; KNL_X32-LABEL: test9: +; KNL_X32: ## BB#0: +; KNL_X32-NEXT: vmovsd {{[0-9]+}}(%esp), %xmm0 +; KNL_X32-NEXT: vucomisd {{[0-9]+}}(%esp), %xmm0 +; KNL_X32-NEXT: setb %al +; KNL_X32-NEXT: retl %c = fcmp ugt double %a, %b ret i1 %c } -; KNL_X32-LABEL: test10 -; KNL_X32: testb $1, 12(%esp) -; KNL_X32: cmovnel - -; KNL-LABEL: test10 -; KNL: testb $1, %dl -; KNL: cmovel define i32 @test10(i32 %a, i32 %b, i1 %cond) { +; ALL_X64-LABEL: test10: +; ALL_X64: ## BB#0: +; ALL_X64-NEXT: testb $1, %dl +; ALL_X64-NEXT: cmovel %esi, %edi +; ALL_X64-NEXT: movl %edi, %eax +; ALL_X64-NEXT: retq +; +; KNL_X32-LABEL: test10: +; KNL_X32: ## BB#0: +; KNL_X32-NEXT: testb $1, {{[0-9]+}}(%esp) +; KNL_X32-NEXT: leal {{[0-9]+}}(%esp), %eax +; KNL_X32-NEXT: leal {{[0-9]+}}(%esp), %ecx +; KNL_X32-NEXT: cmovnel %eax, %ecx +; KNL_X32-NEXT: movl (%ecx), %eax +; KNL_X32-NEXT: retl %c = select i1 %cond, i32 %a, i32 %b ret i32 %c } -; KNL-LABEL: test11 -; KNL: cmp -; KNL: setg define i1 @test11(i32 %a, i32 %b) { +; ALL_X64-LABEL: test11: +; ALL_X64: ## BB#0: +; ALL_X64-NEXT: cmpl %esi, %edi +; ALL_X64-NEXT: setg %al +; ALL_X64-NEXT: retq +; +; KNL_X32-LABEL: test11: +; KNL_X32: ## BB#0: +; KNL_X32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_X32-NEXT: cmpl {{[0-9]+}}(%esp), %eax +; KNL_X32-NEXT: setg %al +; KNL_X32-NEXT: retl %c = icmp sgt i32 %a, %b ret i1 %c } -; KNL-LABEL: test12 -; KNL: callq _test11 -;; return value in %al -; KNL: movzbl %al, %ebx -; KNL: callq _test10 -; KNL: testb $1, %bl - define i32 @test12(i32 %a1, i32 %a2, i32 %b1) { +; ALL_X64-LABEL: test12: +; ALL_X64: ## BB#0: +; ALL_X64-NEXT: pushq %rbp +; ALL_X64-NEXT: Ltmp4: +; ALL_X64-NEXT: .cfi_def_cfa_offset 16 +; ALL_X64-NEXT: pushq %r14 +; ALL_X64-NEXT: Ltmp5: +; ALL_X64-NEXT: .cfi_def_cfa_offset 24 +; ALL_X64-NEXT: pushq %rbx +; ALL_X64-NEXT: Ltmp6: +; ALL_X64-NEXT: .cfi_def_cfa_offset 32 +; ALL_X64-NEXT: Ltmp7: +; ALL_X64-NEXT: .cfi_offset %rbx, -32 +; ALL_X64-NEXT: Ltmp8: +; ALL_X64-NEXT: .cfi_offset %r14, -24 +; ALL_X64-NEXT: Ltmp9: +; ALL_X64-NEXT: .cfi_offset %rbp, -16 +; ALL_X64-NEXT: movl %esi, %r14d +; ALL_X64-NEXT: movl %edi, %ebp +; ALL_X64-NEXT: movl %edx, %esi +; ALL_X64-NEXT: callq _test11 +; ALL_X64-NEXT: movzbl %al, %ebx +; ALL_X64-NEXT: movl %ebp, %edi +; ALL_X64-NEXT: movl %r14d, %esi +; ALL_X64-NEXT: movl %ebx, %edx +; ALL_X64-NEXT: callq _test10 +; ALL_X64-NEXT: xorl %ecx, %ecx +; ALL_X64-NEXT: testb $1, %bl +; ALL_X64-NEXT: cmovel %ecx, %eax +; ALL_X64-NEXT: popq %rbx +; ALL_X64-NEXT: popq %r14 +; ALL_X64-NEXT: popq %rbp +; ALL_X64-NEXT: retq +; +; KNL_X32-LABEL: test12: +; KNL_X32: ## BB#0: +; KNL_X32-NEXT: pushl %ebx +; KNL_X32-NEXT: Ltmp4: +; KNL_X32-NEXT: .cfi_def_cfa_offset 8 +; KNL_X32-NEXT: pushl %edi +; KNL_X32-NEXT: Ltmp5: +; KNL_X32-NEXT: .cfi_def_cfa_offset 12 +; KNL_X32-NEXT: pushl %esi +; KNL_X32-NEXT: Ltmp6: +; KNL_X32-NEXT: .cfi_def_cfa_offset 16 +; KNL_X32-NEXT: subl $16, %esp +; KNL_X32-NEXT: Ltmp7: +; KNL_X32-NEXT: .cfi_def_cfa_offset 32 +; KNL_X32-NEXT: Ltmp8: +; KNL_X32-NEXT: .cfi_offset %esi, -16 +; KNL_X32-NEXT: Ltmp9: +; KNL_X32-NEXT: .cfi_offset %edi, -12 +; KNL_X32-NEXT: Ltmp10: +; KNL_X32-NEXT: .cfi_offset %ebx, -8 +; KNL_X32-NEXT: movl {{[0-9]+}}(%esp), %esi +; KNL_X32-NEXT: movl {{[0-9]+}}(%esp), %edi +; KNL_X32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_X32-NEXT: movl %eax, {{[0-9]+}}(%esp) +; KNL_X32-NEXT: movl %edi, (%esp) +; KNL_X32-NEXT: calll _test11 +; KNL_X32-NEXT: movb %al, %bl +; KNL_X32-NEXT: movzbl %bl, %eax +; KNL_X32-NEXT: movl %eax, {{[0-9]+}}(%esp) +; KNL_X32-NEXT: movl %esi, {{[0-9]+}}(%esp) +; KNL_X32-NEXT: movl %edi, (%esp) +; KNL_X32-NEXT: calll _test10 +; KNL_X32-NEXT: xorl %ecx, %ecx +; KNL_X32-NEXT: testb $1, %bl +; KNL_X32-NEXT: cmovel %ecx, %eax +; KNL_X32-NEXT: addl $16, %esp +; KNL_X32-NEXT: popl %esi +; KNL_X32-NEXT: popl %edi +; KNL_X32-NEXT: popl %ebx +; KNL_X32-NEXT: retl %cond = call i1 @test11(i32 %a1, i32 %b1) %res = call i32 @test10(i32 %a1, i32 %a2, i1 %cond) %res1 = select i1 %cond, i32 %res, i32 0 ret i32 %res1 -} \ No newline at end of file +} diff --git a/test/CodeGen/X86/avx512-cvt.ll b/test/CodeGen/X86/avx512-cvt.ll index a211bcd38c9c..586a29545014 100644 --- a/test/CodeGen/X86/avx512-cvt.ll +++ b/test/CodeGen/X86/avx512-cvt.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl --show-mc-encoding | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx --show-mc-encoding | FileCheck %s ; CHECK-LABEL: sitof32 ; CHECK: vcvtdq2ps %zmm @@ -8,6 +8,70 @@ define <16 x float> @sitof32(<16 x i32> %a) nounwind { ret <16 x float> %b } +; CHECK-LABEL: sltof864 +; CHECK: vcvtqq2pd +define <8 x double> @sltof864(<8 x i64> %a) { + %b = sitofp <8 x i64> %a to <8 x double> + ret <8 x double> %b +} + +; CHECK-LABEL: sltof464 +; CHECK: vcvtqq2pd +define <4 x double> @sltof464(<4 x i64> %a) { + %b = sitofp <4 x i64> %a to <4 x double> + ret <4 x double> %b +} + +; CHECK-LABEL: sltof2f32 +; CHECK: vcvtqq2ps +define <2 x float> @sltof2f32(<2 x i64> %a) { + %b = sitofp <2 x i64> %a to <2 x float> + ret <2 x float>%b +} + +; CHECK-LABEL: sltof4f32_mem +; CHECK: vcvtqq2psy (%rdi) +define <4 x float> @sltof4f32_mem(<4 x i64>* %a) { + %a1 = load <4 x i64>, <4 x i64>* %a, align 8 + %b = sitofp <4 x i64> %a1 to <4 x float> + ret <4 x float>%b +} + +; CHECK-LABEL: f64tosl +; CHECK: vcvttpd2qq +define <4 x i64> @f64tosl(<4 x double> %a) { + %b = fptosi <4 x double> %a to <4 x i64> + ret <4 x i64> %b +} + +; CHECK-LABEL: f32tosl +; CHECK: vcvttps2qq +define <4 x i64> @f32tosl(<4 x float> %a) { + %b = fptosi <4 x float> %a to <4 x i64> + ret <4 x i64> %b +} + +; CHECK-LABEL: sltof432 +; CHECK: vcvtqq2ps +define <4 x float> @sltof432(<4 x i64> %a) { + %b = sitofp <4 x i64> %a to <4 x float> + ret <4 x float> %b +} + +; CHECK-LABEL: ultof432 +; CHECK: vcvtuqq2ps +define <4 x float> @ultof432(<4 x i64> %a) { + %b = uitofp <4 x i64> %a to <4 x float> + ret <4 x float> %b +} + +; CHECK-LABEL: ultof64 +; CHECK: vcvtuqq2pd +define <8 x double> @ultof64(<8 x i64> %a) { + %b = uitofp <8 x i64> %a to <8 x double> + ret <8 x double> %b +} + ; CHECK-LABEL: fptosi00 ; CHECK: vcvttps2dq %zmm ; CHECK: ret @@ -64,16 +128,39 @@ define <8 x i32> @fptosi01(<8 x double> %a) { ret <8 x i32> %b } +; CHECK-LABEL: fptosi03 +; CHECK: vcvttpd2dq %ymm +; CHECK: ret +define <4 x i32> @fptosi03(<4 x double> %a) { + %b = fptosi <4 x double> %a to <4 x i32> + ret <4 x i32> %b +} + ; CHECK-LABEL: fptrunc00 ; CHECK: vcvtpd2ps %zmm ; CHECK-NEXT: vcvtpd2ps %zmm -; CHECK-NEXT: vinsertf64x4 $1 +; CHECK-NEXT: vinsertf ; CHECK: ret define <16 x float> @fptrunc00(<16 x double> %b) nounwind { %a = fptrunc <16 x double> %b to <16 x float> ret <16 x float> %a } +; CHECK-LABEL: fptrunc01 +; CHECK: vcvtpd2ps %ymm +define <4 x float> @fptrunc01(<4 x double> %b) { + %a = fptrunc <4 x double> %b to <4 x float> + ret <4 x float> %a +} + +; CHECK-LABEL: fptrunc02 +; CHECK: vcvtpd2ps %ymm0, %xmm0 {%k1} {z} +define <4 x float> @fptrunc02(<4 x double> %b, <4 x i1> %mask) { + %a = fptrunc <4 x double> %b to <4 x float> + %c = select <4 x i1>%mask, <4 x float>%a, <4 x float> zeroinitializer + ret <4 x float> %c +} + ; CHECK-LABEL: fpext00 ; CHECK: vcvtps2pd %ymm0, %zmm0 ; CHECK: ret @@ -82,6 +169,16 @@ define <8 x double> @fpext00(<8 x float> %b) nounwind { ret <8 x double> %a } +; CHECK-LABEL: fpext01 +; CHECK: vcvtps2pd %xmm0, %ymm0 {%k1} {z} +; CHECK: ret +define <4 x double> @fpext01(<4 x float> %b, <4 x double>%b1, <4 x double>%a1) { + %a = fpext <4 x float> %b to <4 x double> + %mask = fcmp ogt <4 x double>%a1, %b1 + %c = select <4 x i1>%mask, <4 x double>%a, <4 x double>zeroinitializer + ret <4 x double> %c +} + ; CHECK-LABEL: funcA ; CHECK: vcvtsi2sdq (%rdi){{.*}} encoding: [0x62 ; CHECK: ret @@ -182,12 +279,14 @@ define i32 @float_to_int(float %x) { ret i32 %res } -; CHECK-LABEL: uitof64 -; CHECK: vcvtudq2pd -; CHECK: vextracti64x4 -; CHECK: vcvtudq2pd -; CHECK: ret define <16 x double> @uitof64(<16 x i32> %a) nounwind { +; CHECK-LABEL: uitof64: +; CHECK: ## BB#0: +; CHECK-NEXT: vcvtudq2pd %ymm0, %zmm2 +; CHECK-NEXT: vextracti32x8 $1, %zmm0, %ymm0 +; CHECK-NEXT: vcvtudq2pd %ymm0, %zmm1 +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %b = uitofp <16 x i32> %a to <16 x double> ret <16 x double> %b } @@ -257,7 +356,7 @@ define double @uitofp03(i32 %a) nounwind { } ; CHECK-LABEL: @sitofp_16i1_float -; CHECK: vpbroadcastd +; CHECK: vpmovm2d ; CHECK: vcvtdq2ps define <16 x float> @sitofp_16i1_float(<16 x i32> %a) { %mask = icmp slt <16 x i32> %a, zeroinitializer @@ -301,7 +400,7 @@ define <8 x double> @sitofp_8i8_double(<8 x i8> %a) { ; CHECK-LABEL: @sitofp_8i1_double -; CHECK: vpbroadcastq +; CHECK: vpmovm2d ; CHECK: vcvtdq2pd define <8 x double> @sitofp_8i1_double(<8 x double> %a) { %cmpres = fcmp ogt <8 x double> %a, zeroinitializer @@ -310,7 +409,7 @@ define <8 x double> @sitofp_8i1_double(<8 x double> %a) { } ; CHECK-LABEL: @uitofp_16i8 -; CHECK: vpmovzxbd +; CHECK: vpmovzxbd ; CHECK: vcvtudq2ps define <16 x float> @uitofp_16i8(<16 x i8>%a) { %b = uitofp <16 x i8> %a to <16 x float> diff --git a/test/CodeGen/X86/avx512-ext.ll b/test/CodeGen/X86/avx512-ext.ll new file mode 100644 index 000000000000..bc1509684475 --- /dev/null +++ b/test/CodeGen/X86/avx512-ext.ll @@ -0,0 +1,1835 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s --check-prefix=ALL --check-prefix=KNL +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s --check-prefix=ALL --check-prefix=SKX + +define <8 x i16> @zext_8x8mem_to_8x16(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_8x8mem_to_8x16: +; KNL: ## BB#0: +; KNL-NEXT: vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero +; KNL-NEXT: vpsllw $15, %xmm0, %xmm0 +; KNL-NEXT: vpsraw $15, %xmm0, %xmm0 +; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: zext_8x8mem_to_8x16: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $15, %xmm0, %xmm0 +; SKX-NEXT: vpmovw2m %xmm0, %k1 +; SKX-NEXT: vpmovzxbw (%rdi), %xmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <8 x i8>,<8 x i8> *%i,align 1 + %x = zext <8 x i8> %a to <8 x i16> + %ret = select <8 x i1> %mask, <8 x i16> %x, <8 x i16> zeroinitializer + ret <8 x i16> %ret +} + +define <8 x i16> @sext_8x8mem_to_8x16(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_8x8mem_to_8x16: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxbw (%rdi), %xmm1 +; KNL-NEXT: vpsllw $15, %xmm0, %xmm0 +; KNL-NEXT: vpsraw $15, %xmm0, %xmm0 +; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: sext_8x8mem_to_8x16: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $15, %xmm0, %xmm0 +; SKX-NEXT: vpmovw2m %xmm0, %k1 +; SKX-NEXT: vpmovsxbw (%rdi), %xmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <8 x i8>,<8 x i8> *%i,align 1 + %x = sext <8 x i8> %a to <8 x i16> + %ret = select <8 x i1> %mask, <8 x i16> %x, <8 x i16> zeroinitializer + ret <8 x i16> %ret +} + + +define <16 x i16> @zext_16x8mem_to_16x16(<16 x i8> *%i , <16 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_16x8mem_to_16x16: +; KNL: ## BB#0: +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero +; KNL-NEXT: vpsllw $15, %ymm0, %ymm0 +; KNL-NEXT: vpsraw $15, %ymm0, %ymm0 +; KNL-NEXT: vpand %ymm1, %ymm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: zext_16x8mem_to_16x16: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm0, %xmm0 +; SKX-NEXT: vpmovb2m %xmm0, %k1 +; SKX-NEXT: vpmovzxbw (%rdi), %ymm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <16 x i8>,<16 x i8> *%i,align 1 + %x = zext <16 x i8> %a to <16 x i16> + %ret = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> zeroinitializer + ret <16 x i16> %ret +} + +define <16 x i16> @sext_16x8mem_to_16x16(<16 x i8> *%i , <16 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_16x8mem_to_16x16: +; KNL: ## BB#0: +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; KNL-NEXT: vpmovsxbw (%rdi), %ymm1 +; KNL-NEXT: vpsllw $15, %ymm0, %ymm0 +; KNL-NEXT: vpsraw $15, %ymm0, %ymm0 +; KNL-NEXT: vpand %ymm1, %ymm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: sext_16x8mem_to_16x16: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm0, %xmm0 +; SKX-NEXT: vpmovb2m %xmm0, %k1 +; SKX-NEXT: vpmovsxbw (%rdi), %ymm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <16 x i8>,<16 x i8> *%i,align 1 + %x = sext <16 x i8> %a to <16 x i16> + %ret = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> zeroinitializer + ret <16 x i16> %ret +} + +define <16 x i16> @zext_16x8_to_16x16(<16 x i8> %a ) nounwind readnone { +; KNL-LABEL: zext_16x8_to_16x16: +; KNL: ## BB#0: +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; KNL-NEXT: retq +; +; SKX-LABEL: zext_16x8_to_16x16: +; SKX: ## BB#0: +; SKX-NEXT: vpmovzxbw %xmm0, %ymm0 +; SKX-NEXT: retq + %x = zext <16 x i8> %a to <16 x i16> + ret <16 x i16> %x +} + +define <16 x i16> @zext_16x8_to_16x16_mask(<16 x i8> %a ,<16 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_16x8_to_16x16_mask: +; KNL: ## BB#0: +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; KNL-NEXT: vpsllw $15, %ymm1, %ymm1 +; KNL-NEXT: vpsraw $15, %ymm1, %ymm1 +; KNL-NEXT: vpand %ymm0, %ymm1, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: zext_16x8_to_16x16_mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm1, %xmm1 +; SKX-NEXT: vpmovb2m %xmm1, %k1 +; SKX-NEXT: vpmovzxbw %xmm0, %ymm0 {%k1} {z} +; SKX-NEXT: retq + %x = zext <16 x i8> %a to <16 x i16> + %ret = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> zeroinitializer + ret <16 x i16> %ret +} + +define <16 x i16> @sext_16x8_to_16x16(<16 x i8> %a ) nounwind readnone { +; ALL-LABEL: sext_16x8_to_16x16: +; ALL: ## BB#0: +; ALL-NEXT: vpmovsxbw %xmm0, %ymm0 +; ALL-NEXT: retq + %x = sext <16 x i8> %a to <16 x i16> + ret <16 x i16> %x +} + +define <16 x i16> @sext_16x8_to_16x16_mask(<16 x i8> %a ,<16 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_16x8_to_16x16_mask: +; KNL: ## BB#0: +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; KNL-NEXT: vpmovsxbw %xmm0, %ymm0 +; KNL-NEXT: vpsllw $15, %ymm1, %ymm1 +; KNL-NEXT: vpsraw $15, %ymm1, %ymm1 +; KNL-NEXT: vpand %ymm0, %ymm1, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: sext_16x8_to_16x16_mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm1, %xmm1 +; SKX-NEXT: vpmovb2m %xmm1, %k1 +; SKX-NEXT: vpmovsxbw %xmm0, %ymm0 {%k1} {z} +; SKX-NEXT: retq + %x = sext <16 x i8> %a to <16 x i16> + %ret = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> zeroinitializer + ret <16 x i16> %ret +} + +define <32 x i16> @zext_32x8mem_to_32x16(<32 x i8> *%i , <32 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_32x8mem_to_32x16: +; KNL: ## BB#0: +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; KNL-NEXT: vpsllw $15, %ymm3, %ymm3 +; KNL-NEXT: vpsraw $15, %ymm3, %ymm3 +; KNL-NEXT: vpand %ymm2, %ymm3, %ymm2 +; KNL-NEXT: vextracti128 $1, %ymm0, %xmm0 +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; KNL-NEXT: vpsllw $15, %ymm0, %ymm0 +; KNL-NEXT: vpsraw $15, %ymm0, %ymm0 +; KNL-NEXT: vpand %ymm1, %ymm0, %ymm1 +; KNL-NEXT: vmovaps %zmm2, %zmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: zext_32x8mem_to_32x16: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %ymm0, %ymm0 +; SKX-NEXT: vpmovb2m %ymm0, %k1 +; SKX-NEXT: vpmovzxbw (%rdi), %zmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <32 x i8>,<32 x i8> *%i,align 1 + %x = zext <32 x i8> %a to <32 x i16> + %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer + ret <32 x i16> %ret +} + +define <32 x i16> @sext_32x8mem_to_32x16(<32 x i8> *%i , <32 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_32x8mem_to_32x16: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxbw 16(%rdi), %ymm1 +; KNL-NEXT: vpmovsxbw (%rdi), %ymm2 +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; KNL-NEXT: vpsllw $15, %ymm3, %ymm3 +; KNL-NEXT: vpsraw $15, %ymm3, %ymm3 +; KNL-NEXT: vpand %ymm2, %ymm3, %ymm2 +; KNL-NEXT: vextracti128 $1, %ymm0, %xmm0 +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; KNL-NEXT: vpsllw $15, %ymm0, %ymm0 +; KNL-NEXT: vpsraw $15, %ymm0, %ymm0 +; KNL-NEXT: vpand %ymm1, %ymm0, %ymm1 +; KNL-NEXT: vmovaps %zmm2, %zmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: sext_32x8mem_to_32x16: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %ymm0, %ymm0 +; SKX-NEXT: vpmovb2m %ymm0, %k1 +; SKX-NEXT: vpmovsxbw (%rdi), %zmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <32 x i8>,<32 x i8> *%i,align 1 + %x = sext <32 x i8> %a to <32 x i16> + %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer + ret <32 x i16> %ret +} + +define <32 x i16> @zext_32x8_to_32x16(<32 x i8> %a ) nounwind readnone { +; KNL-LABEL: zext_32x8_to_32x16: +; KNL: ## BB#0: +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; KNL-NEXT: vextracti128 $1, %ymm0, %xmm0 +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; KNL-NEXT: vmovaps %zmm2, %zmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: zext_32x8_to_32x16: +; SKX: ## BB#0: +; SKX-NEXT: vpmovzxbw %ymm0, %zmm0 +; SKX-NEXT: retq + %x = zext <32 x i8> %a to <32 x i16> + ret <32 x i16> %x +} + +define <32 x i16> @zext_32x8_to_32x16_mask(<32 x i8> %a ,<32 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_32x8_to_32x16_mask: +; KNL: ## BB#0: +; KNL-NEXT: vextracti128 $1, %ymm0, %xmm2 +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; KNL-NEXT: vpsllw $15, %ymm3, %ymm3 +; KNL-NEXT: vpsraw $15, %ymm3, %ymm3 +; KNL-NEXT: vpand %ymm0, %ymm3, %ymm0 +; KNL-NEXT: vextracti128 $1, %ymm1, %xmm1 +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; KNL-NEXT: vpsllw $15, %ymm1, %ymm1 +; KNL-NEXT: vpsraw $15, %ymm1, %ymm1 +; KNL-NEXT: vpand %ymm2, %ymm1, %ymm1 +; KNL-NEXT: retq +; +; SKX-LABEL: zext_32x8_to_32x16_mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %ymm1, %ymm1 +; SKX-NEXT: vpmovb2m %ymm1, %k1 +; SKX-NEXT: vpmovzxbw %ymm0, %zmm0 {%k1} {z} +; SKX-NEXT: retq + %x = zext <32 x i8> %a to <32 x i16> + %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer + ret <32 x i16> %ret +} + +define <32 x i16> @sext_32x8_to_32x16(<32 x i8> %a ) nounwind readnone { +; KNL-LABEL: sext_32x8_to_32x16: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxbw %xmm0, %ymm2 +; KNL-NEXT: vextracti128 $1, %ymm0, %xmm0 +; KNL-NEXT: vpmovsxbw %xmm0, %ymm1 +; KNL-NEXT: vmovaps %zmm2, %zmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: sext_32x8_to_32x16: +; SKX: ## BB#0: +; SKX-NEXT: vpmovsxbw %ymm0, %zmm0 +; SKX-NEXT: retq + %x = sext <32 x i8> %a to <32 x i16> + ret <32 x i16> %x +} + +define <32 x i16> @sext_32x8_to_32x16_mask(<32 x i8> %a ,<32 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_32x8_to_32x16_mask: +; KNL: ## BB#0: +; KNL-NEXT: vextracti128 $1, %ymm0, %xmm2 +; KNL-NEXT: vpmovsxbw %xmm2, %ymm2 +; KNL-NEXT: vpmovsxbw %xmm0, %ymm0 +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; KNL-NEXT: vpsllw $15, %ymm3, %ymm3 +; KNL-NEXT: vpsraw $15, %ymm3, %ymm3 +; KNL-NEXT: vpand %ymm0, %ymm3, %ymm0 +; KNL-NEXT: vextracti128 $1, %ymm1, %xmm1 +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero +; KNL-NEXT: vpsllw $15, %ymm1, %ymm1 +; KNL-NEXT: vpsraw $15, %ymm1, %ymm1 +; KNL-NEXT: vpand %ymm2, %ymm1, %ymm1 +; KNL-NEXT: retq +; +; SKX-LABEL: sext_32x8_to_32x16_mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %ymm1, %ymm1 +; SKX-NEXT: vpmovb2m %ymm1, %k1 +; SKX-NEXT: vpmovsxbw %ymm0, %zmm0 {%k1} {z} +; SKX-NEXT: retq + %x = sext <32 x i8> %a to <32 x i16> + %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer + ret <32 x i16> %ret +} + +define <4 x i32> @zext_4x8mem_to_4x32(<4 x i8> *%i , <4 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_4x8mem_to_4x32: +; KNL: ## BB#0: +; KNL-NEXT: vpslld $31, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero +; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: zext_4x8mem_to_4x32: +; SKX: ## BB#0: +; SKX-NEXT: vpslld $31, %xmm0, %xmm0 +; SKX-NEXT: vpmovd2m %xmm0, %k1 +; SKX-NEXT: vpmovzxbd (%rdi), %xmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <4 x i8>,<4 x i8> *%i,align 1 + %x = zext <4 x i8> %a to <4 x i32> + %ret = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer + ret <4 x i32> %ret +} + +define <4 x i32> @sext_4x8mem_to_4x32(<4 x i8> *%i , <4 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_4x8mem_to_4x32: +; KNL: ## BB#0: +; KNL-NEXT: vpslld $31, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: vpmovsxbd (%rdi), %xmm1 +; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: sext_4x8mem_to_4x32: +; SKX: ## BB#0: +; SKX-NEXT: vpslld $31, %xmm0, %xmm0 +; SKX-NEXT: vpmovd2m %xmm0, %k1 +; SKX-NEXT: vpmovsxbd (%rdi), %xmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <4 x i8>,<4 x i8> *%i,align 1 + %x = sext <4 x i8> %a to <4 x i32> + %ret = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer + ret <4 x i32> %ret +} + +define <8 x i32> @zext_8x8mem_to_8x32(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_8x8mem_to_8x32: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwq %xmm0, %zmm0 +; KNL-NEXT: vpsllq $63, %zmm0, %zmm0 +; KNL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; KNL-NEXT: vpmovzxbd {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero +; KNL-NEXT: vpxor %ymm1, %ymm1, %ymm1 +; KNL-NEXT: vpblendmd %zmm0, %zmm1, %zmm0 {%k1} +; KNL-NEXT: retq +; +; SKX-LABEL: zext_8x8mem_to_8x32: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $15, %xmm0, %xmm0 +; SKX-NEXT: vpmovw2m %xmm0, %k1 +; SKX-NEXT: vpmovzxbd (%rdi), %ymm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <8 x i8>,<8 x i8> *%i,align 1 + %x = zext <8 x i8> %a to <8 x i32> + %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer + ret <8 x i32> %ret +} + +define <8 x i32> @sext_8x8mem_to_8x32(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_8x8mem_to_8x32: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwq %xmm0, %zmm0 +; KNL-NEXT: vpsllq $63, %zmm0, %zmm0 +; KNL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; KNL-NEXT: vpmovsxbd (%rdi), %ymm0 +; KNL-NEXT: vpxor %ymm1, %ymm1, %ymm1 +; KNL-NEXT: vpblendmd %zmm0, %zmm1, %zmm0 {%k1} +; KNL-NEXT: retq +; +; SKX-LABEL: sext_8x8mem_to_8x32: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $15, %xmm0, %xmm0 +; SKX-NEXT: vpmovw2m %xmm0, %k1 +; SKX-NEXT: vpmovsxbd (%rdi), %ymm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <8 x i8>,<8 x i8> *%i,align 1 + %x = sext <8 x i8> %a to <8 x i32> + %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer + ret <8 x i32> %ret +} + +define <16 x i32> @zext_16x8mem_to_16x32(<16 x i8> *%i , <16 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_16x8mem_to_16x32: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxbd %xmm0, %zmm0 +; KNL-NEXT: vpslld $31, %zmm0, %zmm0 +; KNL-NEXT: vptestmd %zmm0, %zmm0, %k1 +; KNL-NEXT: vpmovzxbd (%rdi), %zmm0 {%k1} {z} +; KNL-NEXT: retq +; +; SKX-LABEL: zext_16x8mem_to_16x32: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm0, %xmm0 +; SKX-NEXT: vpmovb2m %xmm0, %k1 +; SKX-NEXT: vpmovzxbd (%rdi), %zmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <16 x i8>,<16 x i8> *%i,align 1 + %x = zext <16 x i8> %a to <16 x i32> + %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer + ret <16 x i32> %ret +} + +define <16 x i32> @sext_16x8mem_to_16x32(<16 x i8> *%i , <16 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_16x8mem_to_16x32: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxbd %xmm0, %zmm0 +; KNL-NEXT: vpslld $31, %zmm0, %zmm0 +; KNL-NEXT: vptestmd %zmm0, %zmm0, %k1 +; KNL-NEXT: vpmovsxbd (%rdi), %zmm0 {%k1} {z} +; KNL-NEXT: retq +; +; SKX-LABEL: sext_16x8mem_to_16x32: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm0, %xmm0 +; SKX-NEXT: vpmovb2m %xmm0, %k1 +; SKX-NEXT: vpmovsxbd (%rdi), %zmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <16 x i8>,<16 x i8> *%i,align 1 + %x = sext <16 x i8> %a to <16 x i32> + %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer + ret <16 x i32> %ret +} + +define <16 x i32> @zext_16x8_to_16x32_mask(<16 x i8> %a , <16 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_16x8_to_16x32_mask: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxbd %xmm1, %zmm1 +; KNL-NEXT: vpslld $31, %zmm1, %zmm1 +; KNL-NEXT: vptestmd %zmm1, %zmm1, %k1 +; KNL-NEXT: vpmovzxbd %xmm0, %zmm0 {%k1} {z} +; KNL-NEXT: retq +; +; SKX-LABEL: zext_16x8_to_16x32_mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm1, %xmm1 +; SKX-NEXT: vpmovb2m %xmm1, %k1 +; SKX-NEXT: vpmovzxbd %xmm0, %zmm0 {%k1} {z} +; SKX-NEXT: retq + %x = zext <16 x i8> %a to <16 x i32> + %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer + ret <16 x i32> %ret +} + +define <16 x i32> @sext_16x8_to_16x32_mask(<16 x i8> %a , <16 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_16x8_to_16x32_mask: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxbd %xmm1, %zmm1 +; KNL-NEXT: vpslld $31, %zmm1, %zmm1 +; KNL-NEXT: vptestmd %zmm1, %zmm1, %k1 +; KNL-NEXT: vpmovsxbd %xmm0, %zmm0 {%k1} {z} +; KNL-NEXT: retq +; +; SKX-LABEL: sext_16x8_to_16x32_mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm1, %xmm1 +; SKX-NEXT: vpmovb2m %xmm1, %k1 +; SKX-NEXT: vpmovsxbd %xmm0, %zmm0 {%k1} {z} +; SKX-NEXT: retq + %x = sext <16 x i8> %a to <16 x i32> + %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer + ret <16 x i32> %ret +} + +define <16 x i32> @zext_16x8_to_16x32(<16 x i8> %i) nounwind readnone { +; ALL-LABEL: zext_16x8_to_16x32: +; ALL: ## BB#0: +; ALL-NEXT: vpmovzxbd %xmm0, %zmm0 +; ALL-NEXT: retq + %x = zext <16 x i8> %i to <16 x i32> + ret <16 x i32> %x +} + +define <16 x i32> @sext_16x8_to_16x32(<16 x i8> %i) nounwind readnone { +; ALL-LABEL: sext_16x8_to_16x32: +; ALL: ## BB#0: +; ALL-NEXT: vpmovsxbd %xmm0, %zmm0 +; ALL-NEXT: retq + %x = sext <16 x i8> %i to <16 x i32> + ret <16 x i32> %x +} + +define <2 x i64> @zext_2x8mem_to_2x64(<2 x i8> *%i , <2 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_2x8mem_to_2x64: +; KNL: ## BB#0: +; KNL-NEXT: vpsllq $63, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; KNL-NEXT: vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero +; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: zext_2x8mem_to_2x64: +; SKX: ## BB#0: +; SKX-NEXT: vpsllq $63, %xmm0, %xmm0 +; SKX-NEXT: vpmovq2m %xmm0, %k1 +; SKX-NEXT: vpmovzxbq (%rdi), %xmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <2 x i8>,<2 x i8> *%i,align 1 + %x = zext <2 x i8> %a to <2 x i64> + %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer + ret <2 x i64> %ret +} +define <2 x i64> @sext_2x8mem_to_2x64mask(<2 x i8> *%i , <2 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_2x8mem_to_2x64mask: +; KNL: ## BB#0: +; KNL-NEXT: vpsllq $63, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; KNL-NEXT: vpmovsxbq (%rdi), %xmm1 +; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: sext_2x8mem_to_2x64mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllq $63, %xmm0, %xmm0 +; SKX-NEXT: vpmovq2m %xmm0, %k1 +; SKX-NEXT: vpmovsxbq (%rdi), %xmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <2 x i8>,<2 x i8> *%i,align 1 + %x = sext <2 x i8> %a to <2 x i64> + %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer + ret <2 x i64> %ret +} +define <2 x i64> @sext_2x8mem_to_2x64(<2 x i8> *%i) nounwind readnone { +; ALL-LABEL: sext_2x8mem_to_2x64: +; ALL: ## BB#0: +; ALL-NEXT: vpmovsxbq (%rdi), %xmm0 +; ALL-NEXT: retq + %a = load <2 x i8>,<2 x i8> *%i,align 1 + %x = sext <2 x i8> %a to <2 x i64> + ret <2 x i64> %x +} + +define <4 x i64> @zext_4x8mem_to_4x64(<4 x i8> *%i , <4 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_4x8mem_to_4x64: +; KNL: ## BB#0: +; KNL-NEXT: vpslld $31, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: vpmovsxdq %xmm0, %ymm0 +; KNL-NEXT: vpmovzxbq {{.*#+}} ymm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero +; KNL-NEXT: vpand %ymm1, %ymm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: zext_4x8mem_to_4x64: +; SKX: ## BB#0: +; SKX-NEXT: vpslld $31, %xmm0, %xmm0 +; SKX-NEXT: vpmovd2m %xmm0, %k1 +; SKX-NEXT: vpmovzxbq (%rdi), %ymm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <4 x i8>,<4 x i8> *%i,align 1 + %x = zext <4 x i8> %a to <4 x i64> + %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer + ret <4 x i64> %ret +} + +define <4 x i64> @sext_4x8mem_to_4x64mask(<4 x i8> *%i , <4 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_4x8mem_to_4x64mask: +; KNL: ## BB#0: +; KNL-NEXT: vpslld $31, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: vpmovsxdq %xmm0, %ymm0 +; KNL-NEXT: vpmovsxbq (%rdi), %ymm1 +; KNL-NEXT: vpand %ymm1, %ymm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: sext_4x8mem_to_4x64mask: +; SKX: ## BB#0: +; SKX-NEXT: vpslld $31, %xmm0, %xmm0 +; SKX-NEXT: vpmovd2m %xmm0, %k1 +; SKX-NEXT: vpmovsxbq (%rdi), %ymm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <4 x i8>,<4 x i8> *%i,align 1 + %x = sext <4 x i8> %a to <4 x i64> + %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer + ret <4 x i64> %ret +} + +define <4 x i64> @sext_4x8mem_to_4x64(<4 x i8> *%i) nounwind readnone { +; ALL-LABEL: sext_4x8mem_to_4x64: +; ALL: ## BB#0: +; ALL-NEXT: vpmovsxbq (%rdi), %ymm0 +; ALL-NEXT: retq + %a = load <4 x i8>,<4 x i8> *%i,align 1 + %x = sext <4 x i8> %a to <4 x i64> + ret <4 x i64> %x +} + +define <8 x i64> @zext_8x8mem_to_8x64(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_8x8mem_to_8x64: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwq %xmm0, %zmm0 +; KNL-NEXT: vpsllq $63, %zmm0, %zmm0 +; KNL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; KNL-NEXT: vpmovzxbq (%rdi), %zmm0 {%k1} {z} +; KNL-NEXT: retq +; +; SKX-LABEL: zext_8x8mem_to_8x64: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $15, %xmm0, %xmm0 +; SKX-NEXT: vpmovw2m %xmm0, %k1 +; SKX-NEXT: vpmovzxbq (%rdi), %zmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <8 x i8>,<8 x i8> *%i,align 1 + %x = zext <8 x i8> %a to <8 x i64> + %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer + ret <8 x i64> %ret +} + +define <8 x i64> @sext_8x8mem_to_8x64mask(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_8x8mem_to_8x64mask: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwq %xmm0, %zmm0 +; KNL-NEXT: vpsllq $63, %zmm0, %zmm0 +; KNL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; KNL-NEXT: vpmovsxbq (%rdi), %zmm0 {%k1} {z} +; KNL-NEXT: retq +; +; SKX-LABEL: sext_8x8mem_to_8x64mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $15, %xmm0, %xmm0 +; SKX-NEXT: vpmovw2m %xmm0, %k1 +; SKX-NEXT: vpmovsxbq (%rdi), %zmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <8 x i8>,<8 x i8> *%i,align 1 + %x = sext <8 x i8> %a to <8 x i64> + %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer + ret <8 x i64> %ret +} + +define <8 x i64> @sext_8x8mem_to_8x64(<8 x i8> *%i) nounwind readnone { +; ALL-LABEL: sext_8x8mem_to_8x64: +; ALL: ## BB#0: +; ALL-NEXT: vpmovsxbq (%rdi), %zmm0 +; ALL-NEXT: retq + %a = load <8 x i8>,<8 x i8> *%i,align 1 + %x = sext <8 x i8> %a to <8 x i64> + ret <8 x i64> %x +} + +define <4 x i32> @zext_4x16mem_to_4x32(<4 x i16> *%i , <4 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_4x16mem_to_4x32: +; KNL: ## BB#0: +; KNL-NEXT: vpslld $31, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero +; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: zext_4x16mem_to_4x32: +; SKX: ## BB#0: +; SKX-NEXT: vpslld $31, %xmm0, %xmm0 +; SKX-NEXT: vpmovd2m %xmm0, %k1 +; SKX-NEXT: vpmovzxwd (%rdi), %xmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <4 x i16>,<4 x i16> *%i,align 1 + %x = zext <4 x i16> %a to <4 x i32> + %ret = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer + ret <4 x i32> %ret +} + +define <4 x i32> @sext_4x16mem_to_4x32mask(<4 x i16> *%i , <4 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_4x16mem_to_4x32mask: +; KNL: ## BB#0: +; KNL-NEXT: vpslld $31, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: vpmovsxwd (%rdi), %xmm1 +; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: sext_4x16mem_to_4x32mask: +; SKX: ## BB#0: +; SKX-NEXT: vpslld $31, %xmm0, %xmm0 +; SKX-NEXT: vpmovd2m %xmm0, %k1 +; SKX-NEXT: vpmovsxwd (%rdi), %xmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <4 x i16>,<4 x i16> *%i,align 1 + %x = sext <4 x i16> %a to <4 x i32> + %ret = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer + ret <4 x i32> %ret +} + +define <4 x i32> @sext_4x16mem_to_4x32(<4 x i16> *%i) nounwind readnone { +; ALL-LABEL: sext_4x16mem_to_4x32: +; ALL: ## BB#0: +; ALL-NEXT: vpmovsxwd (%rdi), %xmm0 +; ALL-NEXT: retq + %a = load <4 x i16>,<4 x i16> *%i,align 1 + %x = sext <4 x i16> %a to <4 x i32> + ret <4 x i32> %x +} + + +define <8 x i32> @zext_8x16mem_to_8x32(<8 x i16> *%i , <8 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_8x16mem_to_8x32: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwq %xmm0, %zmm0 +; KNL-NEXT: vpsllq $63, %zmm0, %zmm0 +; KNL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; KNL-NEXT: vpmovzxwd {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero +; KNL-NEXT: vpxor %ymm1, %ymm1, %ymm1 +; KNL-NEXT: vpblendmd %zmm0, %zmm1, %zmm0 {%k1} +; KNL-NEXT: retq +; +; SKX-LABEL: zext_8x16mem_to_8x32: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $15, %xmm0, %xmm0 +; SKX-NEXT: vpmovw2m %xmm0, %k1 +; SKX-NEXT: vpmovzxwd (%rdi), %ymm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <8 x i16>,<8 x i16> *%i,align 1 + %x = zext <8 x i16> %a to <8 x i32> + %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer + ret <8 x i32> %ret +} + +define <8 x i32> @sext_8x16mem_to_8x32mask(<8 x i16> *%i , <8 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_8x16mem_to_8x32mask: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwq %xmm0, %zmm0 +; KNL-NEXT: vpsllq $63, %zmm0, %zmm0 +; KNL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; KNL-NEXT: vpmovsxwd (%rdi), %ymm0 +; KNL-NEXT: vpxor %ymm1, %ymm1, %ymm1 +; KNL-NEXT: vpblendmd %zmm0, %zmm1, %zmm0 {%k1} +; KNL-NEXT: retq +; +; SKX-LABEL: sext_8x16mem_to_8x32mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $15, %xmm0, %xmm0 +; SKX-NEXT: vpmovw2m %xmm0, %k1 +; SKX-NEXT: vpmovsxwd (%rdi), %ymm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <8 x i16>,<8 x i16> *%i,align 1 + %x = sext <8 x i16> %a to <8 x i32> + %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer + ret <8 x i32> %ret +} + +define <8 x i32> @sext_8x16mem_to_8x32(<8 x i16> *%i) nounwind readnone { +; ALL-LABEL: sext_8x16mem_to_8x32: +; ALL: ## BB#0: +; ALL-NEXT: vpmovsxwd (%rdi), %ymm0 +; ALL-NEXT: retq + %a = load <8 x i16>,<8 x i16> *%i,align 1 + %x = sext <8 x i16> %a to <8 x i32> + ret <8 x i32> %x +} + +define <8 x i32> @zext_8x16_to_8x32mask(<8 x i16> %a , <8 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_8x16_to_8x32mask: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwq %xmm1, %zmm1 +; KNL-NEXT: vpsllq $63, %zmm1, %zmm1 +; KNL-NEXT: vptestmq %zmm1, %zmm1, %k1 +; KNL-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; KNL-NEXT: vpxor %ymm1, %ymm1, %ymm1 +; KNL-NEXT: vpblendmd %zmm0, %zmm1, %zmm0 {%k1} +; KNL-NEXT: retq +; +; SKX-LABEL: zext_8x16_to_8x32mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $15, %xmm1, %xmm1 +; SKX-NEXT: vpmovw2m %xmm1, %k1 +; SKX-NEXT: vpmovzxwd %xmm0, %ymm0 {%k1} {z} +; SKX-NEXT: retq + %x = zext <8 x i16> %a to <8 x i32> + %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer + ret <8 x i32> %ret +} + +define <8 x i32> @zext_8x16_to_8x32(<8 x i16> %a ) nounwind readnone { +; KNL-LABEL: zext_8x16_to_8x32: +; KNL: ## BB#0: +; KNL-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; KNL-NEXT: retq +; +; SKX-LABEL: zext_8x16_to_8x32: +; SKX: ## BB#0: +; SKX-NEXT: vpmovzxwd %xmm0, %ymm0 +; SKX-NEXT: retq + %x = zext <8 x i16> %a to <8 x i32> + ret <8 x i32> %x +} + +define <16 x i32> @zext_16x16mem_to_16x32(<16 x i16> *%i , <16 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_16x16mem_to_16x32: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxbd %xmm0, %zmm0 +; KNL-NEXT: vpslld $31, %zmm0, %zmm0 +; KNL-NEXT: vptestmd %zmm0, %zmm0, %k1 +; KNL-NEXT: vpmovzxwd (%rdi), %zmm0 {%k1} {z} +; KNL-NEXT: retq +; +; SKX-LABEL: zext_16x16mem_to_16x32: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm0, %xmm0 +; SKX-NEXT: vpmovb2m %xmm0, %k1 +; SKX-NEXT: vpmovzxwd (%rdi), %zmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <16 x i16>,<16 x i16> *%i,align 1 + %x = zext <16 x i16> %a to <16 x i32> + %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer + ret <16 x i32> %ret +} + +define <16 x i32> @sext_16x16mem_to_16x32mask(<16 x i16> *%i , <16 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_16x16mem_to_16x32mask: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxbd %xmm0, %zmm0 +; KNL-NEXT: vpslld $31, %zmm0, %zmm0 +; KNL-NEXT: vptestmd %zmm0, %zmm0, %k1 +; KNL-NEXT: vpmovsxwd (%rdi), %zmm0 {%k1} {z} +; KNL-NEXT: retq +; +; SKX-LABEL: sext_16x16mem_to_16x32mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm0, %xmm0 +; SKX-NEXT: vpmovb2m %xmm0, %k1 +; SKX-NEXT: vpmovsxwd (%rdi), %zmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <16 x i16>,<16 x i16> *%i,align 1 + %x = sext <16 x i16> %a to <16 x i32> + %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer + ret <16 x i32> %ret +} + +define <16 x i32> @sext_16x16mem_to_16x32(<16 x i16> *%i) nounwind readnone { +; ALL-LABEL: sext_16x16mem_to_16x32: +; ALL: ## BB#0: +; ALL-NEXT: vpmovsxwd (%rdi), %zmm0 +; ALL-NEXT: retq + %a = load <16 x i16>,<16 x i16> *%i,align 1 + %x = sext <16 x i16> %a to <16 x i32> + ret <16 x i32> %x +} +define <16 x i32> @zext_16x16_to_16x32mask(<16 x i16> %a , <16 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_16x16_to_16x32mask: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxbd %xmm1, %zmm1 +; KNL-NEXT: vpslld $31, %zmm1, %zmm1 +; KNL-NEXT: vptestmd %zmm1, %zmm1, %k1 +; KNL-NEXT: vpmovzxwd %ymm0, %zmm0 {%k1} {z} +; KNL-NEXT: retq +; +; SKX-LABEL: zext_16x16_to_16x32mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm1, %xmm1 +; SKX-NEXT: vpmovb2m %xmm1, %k1 +; SKX-NEXT: vpmovzxwd %ymm0, %zmm0 {%k1} {z} +; SKX-NEXT: retq + %x = zext <16 x i16> %a to <16 x i32> + %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer + ret <16 x i32> %ret +} + +define <16 x i32> @zext_16x16_to_16x32(<16 x i16> %a ) nounwind readnone { +; ALL-LABEL: zext_16x16_to_16x32: +; ALL: ## BB#0: +; ALL-NEXT: vpmovzxwd %ymm0, %zmm0 +; ALL-NEXT: retq + %x = zext <16 x i16> %a to <16 x i32> + ret <16 x i32> %x +} + +define <2 x i64> @zext_2x16mem_to_2x64(<2 x i16> *%i , <2 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_2x16mem_to_2x64: +; KNL: ## BB#0: +; KNL-NEXT: vpsllq $63, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; KNL-NEXT: vpmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero +; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: zext_2x16mem_to_2x64: +; SKX: ## BB#0: +; SKX-NEXT: vpsllq $63, %xmm0, %xmm0 +; SKX-NEXT: vpmovq2m %xmm0, %k1 +; SKX-NEXT: vpmovzxwq (%rdi), %xmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <2 x i16>,<2 x i16> *%i,align 1 + %x = zext <2 x i16> %a to <2 x i64> + %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer + ret <2 x i64> %ret +} + +define <2 x i64> @sext_2x16mem_to_2x64mask(<2 x i16> *%i , <2 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_2x16mem_to_2x64mask: +; KNL: ## BB#0: +; KNL-NEXT: vpsllq $63, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; KNL-NEXT: vpmovsxwq (%rdi), %xmm1 +; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: sext_2x16mem_to_2x64mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllq $63, %xmm0, %xmm0 +; SKX-NEXT: vpmovq2m %xmm0, %k1 +; SKX-NEXT: vpmovsxwq (%rdi), %xmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <2 x i16>,<2 x i16> *%i,align 1 + %x = sext <2 x i16> %a to <2 x i64> + %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer + ret <2 x i64> %ret +} + +define <2 x i64> @sext_2x16mem_to_2x64(<2 x i16> *%i) nounwind readnone { +; ALL-LABEL: sext_2x16mem_to_2x64: +; ALL: ## BB#0: +; ALL-NEXT: vpmovsxwq (%rdi), %xmm0 +; ALL-NEXT: retq + %a = load <2 x i16>,<2 x i16> *%i,align 1 + %x = sext <2 x i16> %a to <2 x i64> + ret <2 x i64> %x +} + +define <4 x i64> @zext_4x16mem_to_4x64(<4 x i16> *%i , <4 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_4x16mem_to_4x64: +; KNL: ## BB#0: +; KNL-NEXT: vpslld $31, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: vpmovsxdq %xmm0, %ymm0 +; KNL-NEXT: vpmovzxwq {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero +; KNL-NEXT: vpand %ymm1, %ymm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: zext_4x16mem_to_4x64: +; SKX: ## BB#0: +; SKX-NEXT: vpslld $31, %xmm0, %xmm0 +; SKX-NEXT: vpmovd2m %xmm0, %k1 +; SKX-NEXT: vpmovzxwq (%rdi), %ymm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <4 x i16>,<4 x i16> *%i,align 1 + %x = zext <4 x i16> %a to <4 x i64> + %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer + ret <4 x i64> %ret +} + +define <4 x i64> @sext_4x16mem_to_4x64mask(<4 x i16> *%i , <4 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_4x16mem_to_4x64mask: +; KNL: ## BB#0: +; KNL-NEXT: vpslld $31, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: vpmovsxdq %xmm0, %ymm0 +; KNL-NEXT: vpmovsxwq (%rdi), %ymm1 +; KNL-NEXT: vpand %ymm1, %ymm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: sext_4x16mem_to_4x64mask: +; SKX: ## BB#0: +; SKX-NEXT: vpslld $31, %xmm0, %xmm0 +; SKX-NEXT: vpmovd2m %xmm0, %k1 +; SKX-NEXT: vpmovsxwq (%rdi), %ymm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <4 x i16>,<4 x i16> *%i,align 1 + %x = sext <4 x i16> %a to <4 x i64> + %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer + ret <4 x i64> %ret +} + +define <4 x i64> @sext_4x16mem_to_4x64(<4 x i16> *%i) nounwind readnone { +; ALL-LABEL: sext_4x16mem_to_4x64: +; ALL: ## BB#0: +; ALL-NEXT: vpmovsxwq (%rdi), %ymm0 +; ALL-NEXT: retq + %a = load <4 x i16>,<4 x i16> *%i,align 1 + %x = sext <4 x i16> %a to <4 x i64> + ret <4 x i64> %x +} + +define <8 x i64> @zext_8x16mem_to_8x64(<8 x i16> *%i , <8 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_8x16mem_to_8x64: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwq %xmm0, %zmm0 +; KNL-NEXT: vpsllq $63, %zmm0, %zmm0 +; KNL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; KNL-NEXT: vpmovzxwq (%rdi), %zmm0 {%k1} {z} +; KNL-NEXT: retq +; +; SKX-LABEL: zext_8x16mem_to_8x64: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $15, %xmm0, %xmm0 +; SKX-NEXT: vpmovw2m %xmm0, %k1 +; SKX-NEXT: vpmovzxwq (%rdi), %zmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <8 x i16>,<8 x i16> *%i,align 1 + %x = zext <8 x i16> %a to <8 x i64> + %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer + ret <8 x i64> %ret +} + +define <8 x i64> @sext_8x16mem_to_8x64mask(<8 x i16> *%i , <8 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_8x16mem_to_8x64mask: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwq %xmm0, %zmm0 +; KNL-NEXT: vpsllq $63, %zmm0, %zmm0 +; KNL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; KNL-NEXT: vpmovsxwq (%rdi), %zmm0 {%k1} {z} +; KNL-NEXT: retq +; +; SKX-LABEL: sext_8x16mem_to_8x64mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $15, %xmm0, %xmm0 +; SKX-NEXT: vpmovw2m %xmm0, %k1 +; SKX-NEXT: vpmovsxwq (%rdi), %zmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <8 x i16>,<8 x i16> *%i,align 1 + %x = sext <8 x i16> %a to <8 x i64> + %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer + ret <8 x i64> %ret +} + +define <8 x i64> @sext_8x16mem_to_8x64(<8 x i16> *%i) nounwind readnone { +; ALL-LABEL: sext_8x16mem_to_8x64: +; ALL: ## BB#0: +; ALL-NEXT: vpmovsxwq (%rdi), %zmm0 +; ALL-NEXT: retq + %a = load <8 x i16>,<8 x i16> *%i,align 1 + %x = sext <8 x i16> %a to <8 x i64> + ret <8 x i64> %x +} + +define <8 x i64> @zext_8x16_to_8x64mask(<8 x i16> %a , <8 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_8x16_to_8x64mask: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwq %xmm1, %zmm1 +; KNL-NEXT: vpsllq $63, %zmm1, %zmm1 +; KNL-NEXT: vptestmq %zmm1, %zmm1, %k1 +; KNL-NEXT: vpmovzxwq %xmm0, %zmm0 {%k1} {z} +; KNL-NEXT: retq +; +; SKX-LABEL: zext_8x16_to_8x64mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $15, %xmm1, %xmm1 +; SKX-NEXT: vpmovw2m %xmm1, %k1 +; SKX-NEXT: vpmovzxwq %xmm0, %zmm0 {%k1} {z} +; SKX-NEXT: retq + %x = zext <8 x i16> %a to <8 x i64> + %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer + ret <8 x i64> %ret +} + +define <8 x i64> @zext_8x16_to_8x64(<8 x i16> %a) nounwind readnone { +; ALL-LABEL: zext_8x16_to_8x64: +; ALL: ## BB#0: +; ALL-NEXT: vpmovzxwq %xmm0, %zmm0 +; ALL-NEXT: retq + %ret = zext <8 x i16> %a to <8 x i64> + ret <8 x i64> %ret +} + +define <2 x i64> @zext_2x32mem_to_2x64(<2 x i32> *%i , <2 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_2x32mem_to_2x64: +; KNL: ## BB#0: +; KNL-NEXT: vpsllq $63, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; KNL-NEXT: vpmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero +; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: zext_2x32mem_to_2x64: +; SKX: ## BB#0: +; SKX-NEXT: vpsllq $63, %xmm0, %xmm0 +; SKX-NEXT: vpmovq2m %xmm0, %k1 +; SKX-NEXT: vpmovzxdq (%rdi), %xmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <2 x i32>,<2 x i32> *%i,align 1 + %x = zext <2 x i32> %a to <2 x i64> + %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer + ret <2 x i64> %ret +} + +define <2 x i64> @sext_2x32mem_to_2x64mask(<2 x i32> *%i , <2 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_2x32mem_to_2x64mask: +; KNL: ## BB#0: +; KNL-NEXT: vpsllq $63, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; KNL-NEXT: vpmovsxdq (%rdi), %xmm1 +; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: sext_2x32mem_to_2x64mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllq $63, %xmm0, %xmm0 +; SKX-NEXT: vpmovq2m %xmm0, %k1 +; SKX-NEXT: vpmovsxdq (%rdi), %xmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <2 x i32>,<2 x i32> *%i,align 1 + %x = sext <2 x i32> %a to <2 x i64> + %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer + ret <2 x i64> %ret +} + +define <2 x i64> @sext_2x32mem_to_2x64(<2 x i32> *%i) nounwind readnone { +; ALL-LABEL: sext_2x32mem_to_2x64: +; ALL: ## BB#0: +; ALL-NEXT: vpmovsxdq (%rdi), %xmm0 +; ALL-NEXT: retq + %a = load <2 x i32>,<2 x i32> *%i,align 1 + %x = sext <2 x i32> %a to <2 x i64> + ret <2 x i64> %x +} + +define <4 x i64> @zext_4x32mem_to_4x64(<4 x i32> *%i , <4 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_4x32mem_to_4x64: +; KNL: ## BB#0: +; KNL-NEXT: vpslld $31, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: vpmovsxdq %xmm0, %ymm0 +; KNL-NEXT: vpmovzxdq {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero +; KNL-NEXT: vpand %ymm1, %ymm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: zext_4x32mem_to_4x64: +; SKX: ## BB#0: +; SKX-NEXT: vpslld $31, %xmm0, %xmm0 +; SKX-NEXT: vpmovd2m %xmm0, %k1 +; SKX-NEXT: vpmovzxdq (%rdi), %ymm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <4 x i32>,<4 x i32> *%i,align 1 + %x = zext <4 x i32> %a to <4 x i64> + %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer + ret <4 x i64> %ret +} + +define <4 x i64> @sext_4x32mem_to_4x64mask(<4 x i32> *%i , <4 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_4x32mem_to_4x64mask: +; KNL: ## BB#0: +; KNL-NEXT: vpslld $31, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: vpmovsxdq %xmm0, %ymm0 +; KNL-NEXT: vpmovsxdq (%rdi), %ymm1 +; KNL-NEXT: vpand %ymm1, %ymm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: sext_4x32mem_to_4x64mask: +; SKX: ## BB#0: +; SKX-NEXT: vpslld $31, %xmm0, %xmm0 +; SKX-NEXT: vpmovd2m %xmm0, %k1 +; SKX-NEXT: vpmovsxdq (%rdi), %ymm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <4 x i32>,<4 x i32> *%i,align 1 + %x = sext <4 x i32> %a to <4 x i64> + %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer + ret <4 x i64> %ret +} + +define <4 x i64> @sext_4x32mem_to_4x64(<4 x i32> *%i) nounwind readnone { +; ALL-LABEL: sext_4x32mem_to_4x64: +; ALL: ## BB#0: +; ALL-NEXT: vpmovsxdq (%rdi), %ymm0 +; ALL-NEXT: retq + %a = load <4 x i32>,<4 x i32> *%i,align 1 + %x = sext <4 x i32> %a to <4 x i64> + ret <4 x i64> %x +} + +define <4 x i64> @sext_4x32_to_4x64(<4 x i32> %a) nounwind readnone { +; ALL-LABEL: sext_4x32_to_4x64: +; ALL: ## BB#0: +; ALL-NEXT: vpmovsxdq %xmm0, %ymm0 +; ALL-NEXT: retq + %x = sext <4 x i32> %a to <4 x i64> + ret <4 x i64> %x +} + +define <4 x i64> @zext_4x32_to_4x64mask(<4 x i32> %a , <4 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_4x32_to_4x64mask: +; KNL: ## BB#0: +; KNL-NEXT: vpslld $31, %xmm1, %xmm1 +; KNL-NEXT: vpsrad $31, %xmm1, %xmm1 +; KNL-NEXT: vpmovsxdq %xmm1, %ymm1 +; KNL-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; KNL-NEXT: vpand %ymm0, %ymm1, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: zext_4x32_to_4x64mask: +; SKX: ## BB#0: +; SKX-NEXT: vpslld $31, %xmm1, %xmm1 +; SKX-NEXT: vpmovd2m %xmm1, %k1 +; SKX-NEXT: vpmovzxdq %xmm0, %ymm0 {%k1} {z} +; SKX-NEXT: retq + %x = zext <4 x i32> %a to <4 x i64> + %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer + ret <4 x i64> %ret +} + +define <8 x i64> @zext_8x32mem_to_8x64(<8 x i32> *%i , <8 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_8x32mem_to_8x64: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwq %xmm0, %zmm0 +; KNL-NEXT: vpsllq $63, %zmm0, %zmm0 +; KNL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; KNL-NEXT: vpmovzxdq (%rdi), %zmm0 {%k1} {z} +; KNL-NEXT: retq +; +; SKX-LABEL: zext_8x32mem_to_8x64: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $15, %xmm0, %xmm0 +; SKX-NEXT: vpmovw2m %xmm0, %k1 +; SKX-NEXT: vpmovzxdq (%rdi), %zmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <8 x i32>,<8 x i32> *%i,align 1 + %x = zext <8 x i32> %a to <8 x i64> + %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer + ret <8 x i64> %ret +} + +define <8 x i64> @sext_8x32mem_to_8x64mask(<8 x i32> *%i , <8 x i1> %mask) nounwind readnone { +; KNL-LABEL: sext_8x32mem_to_8x64mask: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwq %xmm0, %zmm0 +; KNL-NEXT: vpsllq $63, %zmm0, %zmm0 +; KNL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; KNL-NEXT: vpmovsxdq (%rdi), %zmm0 {%k1} {z} +; KNL-NEXT: retq +; +; SKX-LABEL: sext_8x32mem_to_8x64mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $15, %xmm0, %xmm0 +; SKX-NEXT: vpmovw2m %xmm0, %k1 +; SKX-NEXT: vpmovsxdq (%rdi), %zmm0 {%k1} {z} +; SKX-NEXT: retq + %a = load <8 x i32>,<8 x i32> *%i,align 1 + %x = sext <8 x i32> %a to <8 x i64> + %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer + ret <8 x i64> %ret +} + +define <8 x i64> @sext_8x32mem_to_8x64(<8 x i32> *%i) nounwind readnone { +; ALL-LABEL: sext_8x32mem_to_8x64: +; ALL: ## BB#0: +; ALL-NEXT: vpmovsxdq (%rdi), %zmm0 +; ALL-NEXT: retq + %a = load <8 x i32>,<8 x i32> *%i,align 1 + %x = sext <8 x i32> %a to <8 x i64> + ret <8 x i64> %x +} + +define <8 x i64> @sext_8x32_to_8x64(<8 x i32> %a) nounwind readnone { +; ALL-LABEL: sext_8x32_to_8x64: +; ALL: ## BB#0: +; ALL-NEXT: vpmovsxdq %ymm0, %zmm0 +; ALL-NEXT: retq + %x = sext <8 x i32> %a to <8 x i64> + ret <8 x i64> %x +} + +define <8 x i64> @zext_8x32_to_8x64mask(<8 x i32> %a , <8 x i1> %mask) nounwind readnone { +; KNL-LABEL: zext_8x32_to_8x64mask: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwq %xmm1, %zmm1 +; KNL-NEXT: vpsllq $63, %zmm1, %zmm1 +; KNL-NEXT: vptestmq %zmm1, %zmm1, %k1 +; KNL-NEXT: vpmovzxdq %ymm0, %zmm0 {%k1} {z} +; KNL-NEXT: retq +; +; SKX-LABEL: zext_8x32_to_8x64mask: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $15, %xmm1, %xmm1 +; SKX-NEXT: vpmovw2m %xmm1, %k1 +; SKX-NEXT: vpmovzxdq %ymm0, %zmm0 {%k1} {z} +; SKX-NEXT: retq + %x = zext <8 x i32> %a to <8 x i64> + %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer + ret <8 x i64> %ret +} +define <8 x float> @fptrunc_test(<8 x double> %a) nounwind readnone { +; ALL-LABEL: fptrunc_test: +; ALL: ## BB#0: +; ALL-NEXT: vcvtpd2ps %zmm0, %ymm0 +; ALL-NEXT: retq + %b = fptrunc <8 x double> %a to <8 x float> + ret <8 x float> %b +} + +define <8 x double> @fpext_test(<8 x float> %a) nounwind readnone { +; ALL-LABEL: fpext_test: +; ALL: ## BB#0: +; ALL-NEXT: vcvtps2pd %ymm0, %zmm0 +; ALL-NEXT: retq + %b = fpext <8 x float> %a to <8 x double> + ret <8 x double> %b +} + +define <16 x i32> @zext_16i1_to_16xi32(i16 %b) { +; ALL-LABEL: zext_16i1_to_16xi32: +; ALL: ## BB#0: +; ALL-NEXT: kmovw %edi, %k1 +; ALL-NEXT: vpbroadcastd {{.*}}(%rip), %zmm0 {%k1} {z} +; ALL-NEXT: retq + %a = bitcast i16 %b to <16 x i1> + %c = zext <16 x i1> %a to <16 x i32> + ret <16 x i32> %c +} + +define <8 x i64> @zext_8i1_to_8xi64(i8 %b) { +; KNL-LABEL: zext_8i1_to_8xi64: +; KNL: ## BB#0: +; KNL-NEXT: movzbl %dil, %eax +; KNL-NEXT: kmovw %eax, %k1 +; KNL-NEXT: vpbroadcastq {{.*}}(%rip), %zmm0 {%k1} {z} +; KNL-NEXT: retq +; +; SKX-LABEL: zext_8i1_to_8xi64: +; SKX: ## BB#0: +; SKX-NEXT: kmovb %edi, %k1 +; SKX-NEXT: vpbroadcastq {{.*}}(%rip), %zmm0 {%k1} {z} +; SKX-NEXT: retq + %a = bitcast i8 %b to <8 x i1> + %c = zext <8 x i1> %a to <8 x i64> + ret <8 x i64> %c +} + +define i16 @trunc_16i8_to_16i1(<16 x i8> %a) { +; KNL-LABEL: trunc_16i8_to_16i1: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxbd %xmm0, %zmm0 +; KNL-NEXT: vpslld $31, %zmm0, %zmm0 +; KNL-NEXT: vptestmd %zmm0, %zmm0, %k0 +; KNL-NEXT: kmovw %k0, %eax +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_16i8_to_16i1: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm0, %xmm0 +; SKX-NEXT: vpmovb2m %xmm0, %k0 +; SKX-NEXT: kmovw %k0, %eax +; SKX-NEXT: retq + %mask_b = trunc <16 x i8>%a to <16 x i1> + %mask = bitcast <16 x i1> %mask_b to i16 + ret i16 %mask +} + +define i16 @trunc_16i32_to_16i1(<16 x i32> %a) { +; KNL-LABEL: trunc_16i32_to_16i1: +; KNL: ## BB#0: +; KNL-NEXT: vpslld $31, %zmm0, %zmm0 +; KNL-NEXT: vptestmd %zmm0, %zmm0, %k0 +; KNL-NEXT: kmovw %k0, %eax +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_16i32_to_16i1: +; SKX: ## BB#0: +; SKX-NEXT: vpslld $31, %zmm0, %zmm0 +; SKX-NEXT: vpmovd2m %zmm0, %k0 +; SKX-NEXT: kmovw %k0, %eax +; SKX-NEXT: retq + %mask_b = trunc <16 x i32>%a to <16 x i1> + %mask = bitcast <16 x i1> %mask_b to i16 + ret i16 %mask +} + +define <4 x i32> @trunc_4i32_to_4i1(<4 x i32> %a, <4 x i32> %b) { +; KNL-LABEL: trunc_4i32_to_4i1: +; KNL: ## BB#0: +; KNL-NEXT: vpand %xmm1, %xmm0, %xmm0 +; KNL-NEXT: vpslld $31, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_4i32_to_4i1: +; SKX: ## BB#0: +; SKX-NEXT: vpslld $31, %xmm0, %xmm0 +; SKX-NEXT: vpmovd2m %xmm0, %k0 +; SKX-NEXT: vpslld $31, %xmm1, %xmm0 +; SKX-NEXT: vpmovd2m %xmm0, %k1 +; SKX-NEXT: kandw %k1, %k0, %k0 +; SKX-NEXT: vpmovm2d %k0, %xmm0 +; SKX-NEXT: retq + %mask_a = trunc <4 x i32>%a to <4 x i1> + %mask_b = trunc <4 x i32>%b to <4 x i1> + %a_and_b = and <4 x i1>%mask_a, %mask_b + %res = sext <4 x i1>%a_and_b to <4 x i32> + ret <4 x i32>%res +} + + +define i8 @trunc_8i16_to_8i1(<8 x i16> %a) { +; KNL-LABEL: trunc_8i16_to_8i1: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwq %xmm0, %zmm0 +; KNL-NEXT: vpsllq $63, %zmm0, %zmm0 +; KNL-NEXT: vptestmq %zmm0, %zmm0, %k0 +; KNL-NEXT: kmovw %k0, %eax +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_8i16_to_8i1: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $15, %xmm0, %xmm0 +; SKX-NEXT: vpmovw2m %xmm0, %k0 +; SKX-NEXT: kmovb %k0, %eax +; SKX-NEXT: retq + %mask_b = trunc <8 x i16>%a to <8 x i1> + %mask = bitcast <8 x i1> %mask_b to i8 + ret i8 %mask +} + +define <8 x i32> @sext_8i1_8i32(<8 x i32> %a1, <8 x i32> %a2) nounwind { +; KNL-LABEL: sext_8i1_8i32: +; KNL: ## BB#0: +; KNL-NEXT: vpcmpgtd %zmm0, %zmm1, %k0 +; KNL-NEXT: knotw %k0, %k1 +; KNL-NEXT: vpbroadcastq {{.*}}(%rip), %zmm0 {%k1} {z} +; KNL-NEXT: vpmovqd %zmm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: sext_8i1_8i32: +; SKX: ## BB#0: +; SKX-NEXT: vpcmpgtd %ymm0, %ymm1, %k0 +; SKX-NEXT: knotb %k0, %k0 +; SKX-NEXT: vpmovm2d %k0, %ymm0 +; SKX-NEXT: retq + %x = icmp slt <8 x i32> %a1, %a2 + %x1 = xor <8 x i1>%x, + %y = sext <8 x i1> %x1 to <8 x i32> + ret <8 x i32> %y +} + + +define i16 @trunc_i32_to_i1(i32 %a) { +; ALL-LABEL: trunc_i32_to_i1: +; ALL: ## BB#0: +; ALL-NEXT: andl $1, %edi +; ALL-NEXT: kmovw %edi, %k0 +; ALL-NEXT: movw $-4, %ax +; ALL-NEXT: kmovw %eax, %k1 +; ALL-NEXT: korw %k0, %k1, %k0 +; ALL-NEXT: kmovw %k0, %eax +; ALL-NEXT: retq + %a_i = trunc i32 %a to i1 + %maskv = insertelement <16 x i1> , i1 %a_i, i32 0 + %res = bitcast <16 x i1> %maskv to i16 + ret i16 %res +} + +define <8 x i16> @sext_8i1_8i16(<8 x i32> %a1, <8 x i32> %a2) nounwind { +; KNL-LABEL: sext_8i1_8i16: +; KNL: ## BB#0: +; KNL-NEXT: vpcmpgtd %ymm0, %ymm1, %ymm0 +; KNL-NEXT: vpmovdw %zmm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: sext_8i1_8i16: +; SKX: ## BB#0: +; SKX-NEXT: vpcmpgtd %ymm0, %ymm1, %k0 +; SKX-NEXT: vpmovm2w %k0, %xmm0 +; SKX-NEXT: retq + %x = icmp slt <8 x i32> %a1, %a2 + %y = sext <8 x i1> %x to <8 x i16> + ret <8 x i16> %y +} + +define <16 x i32> @sext_16i1_16i32(<16 x i32> %a1, <16 x i32> %a2) nounwind { +; KNL-LABEL: sext_16i1_16i32: +; KNL: ## BB#0: +; KNL-NEXT: vpcmpgtd %zmm0, %zmm1, %k1 +; KNL-NEXT: vpbroadcastd {{.*}}(%rip), %zmm0 {%k1} {z} +; KNL-NEXT: retq +; +; SKX-LABEL: sext_16i1_16i32: +; SKX: ## BB#0: +; SKX-NEXT: vpcmpgtd %zmm0, %zmm1, %k0 +; SKX-NEXT: vpmovm2d %k0, %zmm0 +; SKX-NEXT: retq + %x = icmp slt <16 x i32> %a1, %a2 + %y = sext <16 x i1> %x to <16 x i32> + ret <16 x i32> %y +} + +define <8 x i64> @sext_8i1_8i64(<8 x i32> %a1, <8 x i32> %a2) nounwind { +; KNL-LABEL: sext_8i1_8i64: +; KNL: ## BB#0: +; KNL-NEXT: vpcmpgtd %ymm0, %ymm1, %ymm0 +; KNL-NEXT: vpmovsxdq %ymm0, %zmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: sext_8i1_8i64: +; SKX: ## BB#0: +; SKX-NEXT: vpcmpgtd %ymm0, %ymm1, %k0 +; SKX-NEXT: vpmovm2q %k0, %zmm0 +; SKX-NEXT: retq + %x = icmp slt <8 x i32> %a1, %a2 + %y = sext <8 x i1> %x to <8 x i64> + ret <8 x i64> %y +} + +define void @extload_v8i64(<8 x i8>* %a, <8 x i64>* %res) { +; ALL-LABEL: extload_v8i64: +; ALL: ## BB#0: +; ALL-NEXT: vpmovsxbq (%rdi), %zmm0 +; ALL-NEXT: vmovdqa64 %zmm0, (%rsi) +; ALL-NEXT: retq + %sign_load = load <8 x i8>, <8 x i8>* %a + %c = sext <8 x i8> %sign_load to <8 x i64> + store <8 x i64> %c, <8 x i64>* %res + ret void +} + +define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone { +; KNL-LABEL: test21: +; KNL: ## BB#0: +; KNL-NEXT: pushq %rbp +; KNL-NEXT: pushq %r15 +; KNL-NEXT: pushq %r14 +; KNL-NEXT: pushq %r13 +; KNL-NEXT: pushq %r12 +; KNL-NEXT: pushq %rbx +; KNL-NEXT: vpmovsxbd %xmm7, %zmm7 +; KNL-NEXT: vpslld $31, %zmm7, %zmm7 +; KNL-NEXT: vpmovsxbd %xmm6, %zmm6 +; KNL-NEXT: vpslld $31, %zmm6, %zmm6 +; KNL-NEXT: vpmovsxbd %xmm5, %zmm5 +; KNL-NEXT: vpslld $31, %zmm5, %zmm5 +; KNL-NEXT: vpmovsxbd %xmm4, %zmm4 +; KNL-NEXT: vpslld $31, %zmm4, %zmm4 +; KNL-NEXT: vptestmd %zmm4, %zmm4, %k0 +; KNL-NEXT: kshiftlw $14, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: kmovw %k1, %edx +; KNL-NEXT: kshiftlw $15, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: kmovw %k1, %eax +; KNL-NEXT: kshiftlw $13, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: kmovw %k1, %ecx +; KNL-NEXT: kshiftlw $12, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: kmovw %k1, %edi +; KNL-NEXT: kshiftlw $11, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: kmovw %k1, %esi +; KNL-NEXT: kshiftlw $10, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: kmovw %k1, %r13d +; KNL-NEXT: kshiftlw $9, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: kmovw %k1, %r8d +; KNL-NEXT: kshiftlw $8, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: kmovw %k1, %r10d +; KNL-NEXT: kshiftlw $7, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: kmovw %k1, %r11d +; KNL-NEXT: kshiftlw $6, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: kmovw %k1, %ebx +; KNL-NEXT: kshiftlw $5, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: kmovw %k1, %ebp +; KNL-NEXT: kshiftlw $4, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: kmovw %k1, %r14d +; KNL-NEXT: kshiftlw $3, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: kmovw %k1, %r15d +; KNL-NEXT: kshiftlw $2, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: kmovw %k1, %r9d +; KNL-NEXT: kshiftlw $1, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: kmovw %k1, %r12d +; KNL-NEXT: vptestmd %zmm5, %zmm5, %k1 +; KNL-NEXT: kshiftlw $0, %k0, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vmovd %eax, %xmm4 +; KNL-NEXT: kmovw %k0, %eax +; KNL-NEXT: kshiftlw $14, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $1, %edx, %xmm4, %xmm4 +; KNL-NEXT: kmovw %k0, %edx +; KNL-NEXT: movl %edx, -{{[0-9]+}}(%rsp) ## 4-byte Spill +; KNL-NEXT: kshiftlw $15, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $2, %ecx, %xmm4, %xmm4 +; KNL-NEXT: kmovw %k0, %ecx +; KNL-NEXT: kshiftlw $13, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $3, %edi, %xmm4, %xmm4 +; KNL-NEXT: kmovw %k0, %edi +; KNL-NEXT: kshiftlw $12, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $4, %esi, %xmm4, %xmm4 +; KNL-NEXT: kmovw %k0, %esi +; KNL-NEXT: kshiftlw $11, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $5, %r13d, %xmm4, %xmm4 +; KNL-NEXT: kmovw %k0, %r13d +; KNL-NEXT: kshiftlw $10, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $6, %r8d, %xmm4, %xmm4 +; KNL-NEXT: kmovw %k0, %r8d +; KNL-NEXT: kshiftlw $9, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $7, %r10d, %xmm4, %xmm4 +; KNL-NEXT: kmovw %k0, %r10d +; KNL-NEXT: kshiftlw $8, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $8, %r11d, %xmm4, %xmm4 +; KNL-NEXT: kmovw %k0, %r11d +; KNL-NEXT: kshiftlw $7, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $9, %ebx, %xmm4, %xmm4 +; KNL-NEXT: kmovw %k0, %ebx +; KNL-NEXT: kshiftlw $6, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $10, %ebp, %xmm4, %xmm4 +; KNL-NEXT: kmovw %k0, %ebp +; KNL-NEXT: kshiftlw $5, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $11, %r14d, %xmm4, %xmm4 +; KNL-NEXT: kmovw %k0, %r14d +; KNL-NEXT: kshiftlw $4, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $12, %r15d, %xmm4, %xmm4 +; KNL-NEXT: kmovw %k0, %r15d +; KNL-NEXT: kshiftlw $3, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $13, %r9d, %xmm4, %xmm4 +; KNL-NEXT: kmovw %k0, %edx +; KNL-NEXT: movl %edx, -{{[0-9]+}}(%rsp) ## 4-byte Spill +; KNL-NEXT: kshiftlw $2, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $14, %r12d, %xmm4, %xmm4 +; KNL-NEXT: kmovw %k0, %r12d +; KNL-NEXT: kshiftlw $1, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $15, %eax, %xmm4, %xmm4 +; KNL-NEXT: kmovw %k0, %r9d +; KNL-NEXT: vptestmd %zmm6, %zmm6, %k0 +; KNL-NEXT: kshiftlw $0, %k1, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: vmovd %ecx, %xmm5 +; KNL-NEXT: kmovw %k1, %edx +; KNL-NEXT: kshiftlw $14, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: vpinsrb $1, -{{[0-9]+}}(%rsp), %xmm5, %xmm5 ## 4-byte Folded Reload +; KNL-NEXT: kmovw %k1, %eax +; KNL-NEXT: movl %eax, -{{[0-9]+}}(%rsp) ## 4-byte Spill +; KNL-NEXT: kshiftlw $15, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: vpinsrb $2, %edi, %xmm5, %xmm5 +; KNL-NEXT: kmovw %k1, %eax +; KNL-NEXT: kshiftlw $13, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: vpinsrb $3, %esi, %xmm5, %xmm5 +; KNL-NEXT: kmovw %k1, %edi +; KNL-NEXT: kshiftlw $12, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: vpinsrb $4, %r13d, %xmm5, %xmm5 +; KNL-NEXT: kmovw %k1, %ecx +; KNL-NEXT: kshiftlw $11, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: vpinsrb $5, %r8d, %xmm5, %xmm5 +; KNL-NEXT: kmovw %k1, %r8d +; KNL-NEXT: kshiftlw $10, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: vpinsrb $6, %r10d, %xmm5, %xmm5 +; KNL-NEXT: kmovw %k1, %r13d +; KNL-NEXT: kshiftlw $9, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: vpinsrb $7, %r11d, %xmm5, %xmm5 +; KNL-NEXT: kmovw %k1, %esi +; KNL-NEXT: movl %esi, -{{[0-9]+}}(%rsp) ## 4-byte Spill +; KNL-NEXT: kshiftlw $8, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: vpinsrb $8, %ebx, %xmm5, %xmm5 +; KNL-NEXT: kmovw %k1, %ebx +; KNL-NEXT: kshiftlw $7, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: vpinsrb $9, %ebp, %xmm5, %xmm5 +; KNL-NEXT: kmovw %k1, %ebp +; KNL-NEXT: kshiftlw $6, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: vpinsrb $10, %r14d, %xmm5, %xmm5 +; KNL-NEXT: kmovw %k1, %r10d +; KNL-NEXT: kshiftlw $5, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: vpinsrb $11, %r15d, %xmm5, %xmm5 +; KNL-NEXT: kmovw %k1, %r11d +; KNL-NEXT: kshiftlw $4, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: vpinsrb $12, -{{[0-9]+}}(%rsp), %xmm5, %xmm5 ## 4-byte Folded Reload +; KNL-NEXT: kmovw %k1, %esi +; KNL-NEXT: kshiftlw $3, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: vpinsrb $13, %r12d, %xmm5, %xmm5 +; KNL-NEXT: kmovw %k1, %r14d +; KNL-NEXT: kshiftlw $2, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: vpinsrb $14, %r9d, %xmm5, %xmm5 +; KNL-NEXT: kmovw %k1, %r9d +; KNL-NEXT: kshiftlw $1, %k0, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: vpinsrb $15, %edx, %xmm5, %xmm5 +; KNL-NEXT: kmovw %k1, %r15d +; KNL-NEXT: vptestmd %zmm7, %zmm7, %k1 +; KNL-NEXT: kshiftlw $0, %k0, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vmovd %eax, %xmm6 +; KNL-NEXT: kmovw %k0, %r12d +; KNL-NEXT: kshiftlw $14, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $1, -{{[0-9]+}}(%rsp), %xmm6, %xmm6 ## 4-byte Folded Reload +; KNL-NEXT: kmovw %k0, %eax +; KNL-NEXT: kshiftlw $15, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $2, %edi, %xmm6, %xmm6 +; KNL-NEXT: kmovw %k0, %edx +; KNL-NEXT: kshiftlw $13, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $3, %ecx, %xmm6, %xmm6 +; KNL-NEXT: kmovw %k0, %ecx +; KNL-NEXT: kshiftlw $12, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $4, %r8d, %xmm6, %xmm6 +; KNL-NEXT: kmovw %k0, %r8d +; KNL-NEXT: kshiftlw $11, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $5, %r13d, %xmm6, %xmm6 +; KNL-NEXT: kmovw %k0, %r13d +; KNL-NEXT: kshiftlw $10, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $6, -{{[0-9]+}}(%rsp), %xmm6, %xmm6 ## 4-byte Folded Reload +; KNL-NEXT: kmovw %k0, %edi +; KNL-NEXT: kshiftlw $9, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $7, %ebx, %xmm6, %xmm6 +; KNL-NEXT: kmovw %k0, %ebx +; KNL-NEXT: kshiftlw $8, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $8, %ebp, %xmm6, %xmm6 +; KNL-NEXT: kmovw %k0, %ebp +; KNL-NEXT: kshiftlw $7, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $9, %r10d, %xmm6, %xmm6 +; KNL-NEXT: kmovw %k0, %r10d +; KNL-NEXT: kshiftlw $6, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $10, %r11d, %xmm6, %xmm6 +; KNL-NEXT: kmovw %k0, %r11d +; KNL-NEXT: kshiftlw $5, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $11, %esi, %xmm6, %xmm6 +; KNL-NEXT: kmovw %k0, %esi +; KNL-NEXT: kshiftlw $4, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $12, %r14d, %xmm6, %xmm6 +; KNL-NEXT: kmovw %k0, %r14d +; KNL-NEXT: kshiftlw $3, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $13, %r9d, %xmm6, %xmm6 +; KNL-NEXT: kmovw %k0, %r9d +; KNL-NEXT: kshiftlw $2, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $14, %r15d, %xmm6, %xmm6 +; KNL-NEXT: kmovw %k0, %r15d +; KNL-NEXT: kshiftlw $1, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vpinsrb $15, %r12d, %xmm6, %xmm6 +; KNL-NEXT: kmovw %k0, %r12d +; KNL-NEXT: kshiftlw $0, %k1, %k0 +; KNL-NEXT: kshiftrw $15, %k0, %k0 +; KNL-NEXT: vmovd %edx, %xmm7 +; KNL-NEXT: kmovw %k0, %edx +; KNL-NEXT: vpinsrb $1, %eax, %xmm7, %xmm7 +; KNL-NEXT: vpinsrb $2, %ecx, %xmm7, %xmm7 +; KNL-NEXT: vpinsrb $3, %r8d, %xmm7, %xmm7 +; KNL-NEXT: vpinsrb $4, %r13d, %xmm7, %xmm7 +; KNL-NEXT: vpinsrb $5, %edi, %xmm7, %xmm7 +; KNL-NEXT: vpinsrb $6, %ebx, %xmm7, %xmm7 +; KNL-NEXT: vpinsrb $7, %ebp, %xmm7, %xmm7 +; KNL-NEXT: vpinsrb $8, %r10d, %xmm7, %xmm7 +; KNL-NEXT: vpinsrb $9, %r11d, %xmm7, %xmm7 +; KNL-NEXT: vpinsrb $10, %esi, %xmm7, %xmm7 +; KNL-NEXT: vpinsrb $11, %r14d, %xmm7, %xmm7 +; KNL-NEXT: vpinsrb $12, %r9d, %xmm7, %xmm7 +; KNL-NEXT: vpinsrb $13, %r15d, %xmm7, %xmm7 +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero +; KNL-NEXT: vpsllw $15, %ymm4, %ymm4 +; KNL-NEXT: vpsraw $15, %ymm4, %ymm4 +; KNL-NEXT: vpand %ymm0, %ymm4, %ymm0 +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero +; KNL-NEXT: vpsllw $15, %ymm4, %ymm4 +; KNL-NEXT: vpsraw $15, %ymm4, %ymm4 +; KNL-NEXT: vpand %ymm1, %ymm4, %ymm1 +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero,xmm6[8],zero,xmm6[9],zero,xmm6[10],zero,xmm6[11],zero,xmm6[12],zero,xmm6[13],zero,xmm6[14],zero,xmm6[15],zero +; KNL-NEXT: vpsllw $15, %ymm4, %ymm4 +; KNL-NEXT: vpsraw $15, %ymm4, %ymm4 +; KNL-NEXT: vpand %ymm2, %ymm4, %ymm2 +; KNL-NEXT: vpinsrb $14, %r12d, %xmm7, %xmm4 +; KNL-NEXT: vpinsrb $15, %edx, %xmm4, %xmm4 +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero +; KNL-NEXT: vpsllw $15, %ymm4, %ymm4 +; KNL-NEXT: vpsraw $15, %ymm4, %ymm4 +; KNL-NEXT: vpand %ymm3, %ymm4, %ymm3 +; KNL-NEXT: popq %rbx +; KNL-NEXT: popq %r12 +; KNL-NEXT: popq %r13 +; KNL-NEXT: popq %r14 +; KNL-NEXT: popq %r15 +; KNL-NEXT: popq %rbp +; KNL-NEXT: retq +; +; SKX-LABEL: test21: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %zmm2, %zmm2 +; SKX-NEXT: vpmovb2m %zmm2, %k1 +; SKX-NEXT: vpxord %zmm2, %zmm2, %zmm2 +; SKX-NEXT: vpxord %zmm3, %zmm3, %zmm3 +; SKX-NEXT: vmovdqu16 %zmm0, %zmm3 {%k1} +; SKX-NEXT: kshiftrq $32, %k1, %k1 +; SKX-NEXT: vmovdqu16 %zmm1, %zmm2 {%k1} +; SKX-NEXT: vmovaps %zmm3, %zmm0 +; SKX-NEXT: vmovaps %zmm2, %zmm1 +; SKX-NEXT: retq + %ret = select <64 x i1> %mask, <64 x i16> %x, <64 x i16> zeroinitializer + ret <64 x i16> %ret +} + diff --git a/test/CodeGen/X86/avx512-extract-subvector.ll b/test/CodeGen/X86/avx512-extract-subvector.ll new file mode 100644 index 000000000000..703f7832588c --- /dev/null +++ b/test/CodeGen/X86/avx512-extract-subvector.ll @@ -0,0 +1,56 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck --check-prefix=SKX %s + + +define <8 x i16> @extract_subvector128_v32i16(<32 x i16> %x) nounwind { +; SKX-LABEL: extract_subvector128_v32i16: +; SKX: ## BB#0: +; SKX-NEXT: vextracti32x4 $2, %zmm0, %xmm0 +; SKX-NEXT: retq + %r1 = shufflevector <32 x i16> %x, <32 x i16> undef, <8 x i32> + ret <8 x i16> %r1 +} + +define <8 x i16> @extract_subvector128_v32i16_first_element(<32 x i16> %x) nounwind { +; SKX-LABEL: extract_subvector128_v32i16_first_element: +; SKX: ## BB#0: +; SKX-NEXT: retq + %r1 = shufflevector <32 x i16> %x, <32 x i16> undef, <8 x i32> + ret <8 x i16> %r1 +} + +define <16 x i8> @extract_subvector128_v64i8(<64 x i8> %x) nounwind { +; SKX-LABEL: extract_subvector128_v64i8: +; SKX: ## BB#0: +; SKX-NEXT: vextracti32x4 $2, %zmm0, %xmm0 +; SKX-NEXT: retq + %r1 = shufflevector <64 x i8> %x, <64 x i8> undef, <16 x i32> + ret <16 x i8> %r1 +} + +define <16 x i8> @extract_subvector128_v64i8_first_element(<64 x i8> %x) nounwind { +; SKX-LABEL: extract_subvector128_v64i8_first_element: +; SKX: ## BB#0: +; SKX-NEXT: retq + %r1 = shufflevector <64 x i8> %x, <64 x i8> undef, <16 x i32> + ret <16 x i8> %r1 +} + + +define <16 x i16> @extract_subvector256_v32i16(<32 x i16> %x) nounwind { +; SKX-LABEL: extract_subvector256_v32i16: +; SKX: ## BB#0: +; SKX-NEXT: vextracti64x4 $1, %zmm0, %ymm0 +; SKX-NEXT: retq + %r1 = shufflevector <32 x i16> %x, <32 x i16> undef, <16 x i32> + ret <16 x i16> %r1 +} + +define <32 x i8> @extract_subvector256_v64i8(<64 x i8> %x) nounwind { +; SKX-LABEL: extract_subvector256_v64i8: +; SKX: ## BB#0: +; SKX-NEXT: vextracti64x4 $1, %zmm0, %ymm0 +; SKX-NEXT: retq + %r1 = shufflevector <64 x i8> %x, <64 x i8> undef, <32 x i32> + ret <32 x i8> %r1 +} diff --git a/test/CodeGen/X86/avx512-fma.ll b/test/CodeGen/X86/avx512-fma.ll index ed046de005cf..9279441a23c7 100644 --- a/test/CodeGen/X86/avx512-fma.ll +++ b/test/CodeGen/X86/avx512-fma.ll @@ -1,81 +1,93 @@ -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f -fp-contract=fast | FileCheck %s -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx -fp-contract=fast | FileCheck %s --check-prefix=SKX +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f -fp-contract=fast | FileCheck %s --check-prefix=ALL --check-prefix=KNL +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx -fp-contract=fast | FileCheck %s --check-prefix=ALL --check-prefix=SKX -; CHECK-LABEL: test_x86_fmadd_ps_z -; CHECK: vfmadd213ps %zmm2, %zmm1, %zmm0 -; CHECK: ret define <16 x float> @test_x86_fmadd_ps_z(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) { +; ALL-LABEL: test_x86_fmadd_ps_z: +; ALL: ## BB#0: +; ALL-NEXT: vfmadd213ps %zmm2, %zmm1, %zmm0 +; ALL-NEXT: retq %x = fmul <16 x float> %a0, %a1 %res = fadd <16 x float> %x, %a2 ret <16 x float> %res } -; CHECK-LABEL: test_x86_fmsub_ps_z -; CHECK: vfmsub213ps %zmm2, %zmm1, %zmm0 -; CHECK: ret define <16 x float> @test_x86_fmsub_ps_z(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) { +; ALL-LABEL: test_x86_fmsub_ps_z: +; ALL: ## BB#0: +; ALL-NEXT: vfmsub213ps %zmm2, %zmm1, %zmm0 +; ALL-NEXT: retq %x = fmul <16 x float> %a0, %a1 %res = fsub <16 x float> %x, %a2 ret <16 x float> %res } -; CHECK-LABEL: test_x86_fnmadd_ps_z -; CHECK: vfnmadd213ps %zmm2, %zmm1, %zmm0 -; CHECK: ret define <16 x float> @test_x86_fnmadd_ps_z(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) { +; ALL-LABEL: test_x86_fnmadd_ps_z: +; ALL: ## BB#0: +; ALL-NEXT: vfnmadd213ps %zmm2, %zmm1, %zmm0 +; ALL-NEXT: retq %x = fmul <16 x float> %a0, %a1 %res = fsub <16 x float> %a2, %x ret <16 x float> %res } -; CHECK-LABEL: test_x86_fnmsub_ps_z -; CHECK: vfnmsub213ps %zmm2, %zmm1, %zmm0 -; CHECK: ret define <16 x float> @test_x86_fnmsub_ps_z(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) { +; ALL-LABEL: test_x86_fnmsub_ps_z: +; ALL: ## BB#0: +; ALL-NEXT: vfnmsub213ps %zmm2, %zmm1, %zmm0 +; ALL-NEXT: retq %x = fmul <16 x float> %a0, %a1 - %y = fsub <16 x float> , %x + float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, + float -0.000000e+00>, %x %res = fsub <16 x float> %y, %a2 ret <16 x float> %res } -; CHECK-LABEL: test_x86_fmadd_pd_z -; CHECK: vfmadd213pd %zmm2, %zmm1, %zmm0 -; CHECK: ret define <8 x double> @test_x86_fmadd_pd_z(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) { +; ALL-LABEL: test_x86_fmadd_pd_z: +; ALL: ## BB#0: +; ALL-NEXT: vfmadd213pd %zmm2, %zmm1, %zmm0 +; ALL-NEXT: retq %x = fmul <8 x double> %a0, %a1 %res = fadd <8 x double> %x, %a2 ret <8 x double> %res } -; CHECK-LABEL: test_x86_fmsub_pd_z -; CHECK: vfmsub213pd %zmm2, %zmm1, %zmm0 -; CHECK: ret define <8 x double> @test_x86_fmsub_pd_z(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) { +; ALL-LABEL: test_x86_fmsub_pd_z: +; ALL: ## BB#0: +; ALL-NEXT: vfmsub213pd %zmm2, %zmm1, %zmm0 +; ALL-NEXT: retq %x = fmul <8 x double> %a0, %a1 %res = fsub <8 x double> %x, %a2 ret <8 x double> %res } define double @test_x86_fmsub_213(double %a0, double %a1, double %a2) { -; CHECK-LABEL: test_x86_fmsub_213: -; CHECK: ## BB#0: -; CHECK-NEXT: vfmsub213sd %xmm2, %xmm0, %xmm1 -; CHECK-NEXT: vmovaps %zmm1, %zmm0 -; CHECK-NEXT: retq +; ALL-LABEL: test_x86_fmsub_213: +; ALL: ## BB#0: +; ALL-NEXT: vfmsub213sd %xmm2, %xmm0, %xmm1 +; ALL-NEXT: vmovaps %zmm1, %zmm0 +; ALL-NEXT: retq %x = fmul double %a0, %a1 %res = fsub double %x, %a2 ret double %res } define double @test_x86_fmsub_213_m(double %a0, double %a1, double * %a2_ptr) { -; CHECK-LABEL: test_x86_fmsub_213_m: -; CHECK: ## BB#0: -; CHECK-NEXT: vfmsub213sd (%rdi), %xmm0, %xmm1 -; CHECK-NEXT: vmovaps %zmm1, %zmm0 -; CHECK-NEXT: retq +; KNL-LABEL: test_x86_fmsub_213_m: +; KNL: ## BB#0: +; KNL-NEXT: vfmsub213sd (%rdi), %xmm0, %xmm1 +; KNL-NEXT: vmovaps %zmm1, %zmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test_x86_fmsub_213_m: +; SKX: ## BB#0: +; SKX-NEXT: vfmsub213sd (%rdi), %xmm1, %xmm0 +; SKX-NEXT: retq %a2 = load double , double *%a2_ptr %x = fmul double %a0, %a1 %res = fsub double %x, %a2 @@ -83,11 +95,11 @@ define double @test_x86_fmsub_213_m(double %a0, double %a1, double * %a2_ptr) { } define double @test_x86_fmsub_231_m(double %a0, double %a1, double * %a2_ptr) { -; CHECK-LABEL: test_x86_fmsub_231_m: -; CHECK: ## BB#0: -; CHECK-NEXT: vfmsub231sd (%rdi), %xmm0, %xmm1 -; CHECK-NEXT: vmovaps %zmm1, %zmm0 -; CHECK-NEXT: retq +; ALL-LABEL: test_x86_fmsub_231_m: +; ALL: ## BB#0: +; ALL-NEXT: vfmsub231sd (%rdi), %xmm0, %xmm1 +; ALL-NEXT: vmovaps %zmm1, %zmm0 +; ALL-NEXT: retq %a2 = load double , double *%a2_ptr %x = fmul double %a0, %a2 %res = fsub double %x, %a1 @@ -95,21 +107,21 @@ define double @test_x86_fmsub_231_m(double %a0, double %a1, double * %a2_ptr) { } define <16 x float> @test231_br(<16 x float> %a1, <16 x float> %a2) nounwind { -; CHECK-LABEL: test231_br: -; CHECK: ## BB#0: -; CHECK-NEXT: vfmadd231ps {{.*}}(%rip){1to16}, %zmm0, %zmm1 -; CHECK-NEXT: vmovaps %zmm1, %zmm0 -; CHECK-NEXT: retq +; ALL-LABEL: test231_br: +; ALL: ## BB#0: +; ALL-NEXT: vfmadd231ps {{.*}}(%rip){1to16}, %zmm0, %zmm1 +; ALL-NEXT: vmovaps %zmm1, %zmm0 +; ALL-NEXT: retq %b1 = fmul <16 x float> %a1, %b2 = fadd <16 x float> %b1, %a2 ret <16 x float> %b2 } define <16 x float> @test213_br(<16 x float> %a1, <16 x float> %a2) nounwind { -; CHECK-LABEL: test213_br: -; CHECK: ## BB#0: -; CHECK-NEXT: vfmadd213ps {{.*}}(%rip){1to16}, %zmm1, %zmm0 -; CHECK-NEXT: retq +; ALL-LABEL: test213_br: +; ALL: ## BB#0: +; ALL-NEXT: vfmadd213ps {{.*}}(%rip){1to16}, %zmm1, %zmm0 +; ALL-NEXT: retq %b1 = fmul <16 x float> %a1, %a2 %b2 = fadd <16 x float> %b1, ret <16 x float> %b2 @@ -117,16 +129,17 @@ define <16 x float> @test213_br(<16 x float> %a1, <16 x float> %a2) nounwind { ;mask (a*c+b , a) define <16 x float> @test_x86_fmadd132_ps(<16 x float> %a0, <16 x float> %a1, <16 x float> *%a2_ptrt, <16 x i1> %mask) { -; CHECK-LABEL: test_x86_fmadd132_ps: -; CHECK: ## BB#0: -; CHECK-NEXT: vpmovsxbd %xmm2, %zmm2 -; CHECK-NEXT: vpandd {{.*}}(%rip){1to16}, %zmm2, %zmm2 -; CHECK-NEXT: vptestmd %zmm2, %zmm2, %k1 -; CHECK-NEXT: vfmadd132ps (%rdi), %zmm1, %zmm0 {%k1} -; CHECK-NEXT: retq +; KNL-LABEL: test_x86_fmadd132_ps: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxbd %xmm2, %zmm2 +; KNL-NEXT: vpslld $31, %zmm2, %zmm2 +; KNL-NEXT: vptestmd %zmm2, %zmm2, %k1 +; KNL-NEXT: vfmadd132ps (%rdi), %zmm1, %zmm0 {%k1} +; KNL-NEXT: retq ; ; SKX-LABEL: test_x86_fmadd132_ps: ; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm2, %xmm2 ; SKX-NEXT: vpmovb2m %xmm2, %k1 ; SKX-NEXT: vfmadd132ps (%rdi), %zmm1, %zmm0 {%k1} ; SKX-NEXT: retq @@ -139,17 +152,18 @@ define <16 x float> @test_x86_fmadd132_ps(<16 x float> %a0, <16 x float> %a1, <1 ;mask (a*c+b , b) define <16 x float> @test_x86_fmadd231_ps(<16 x float> %a0, <16 x float> %a1, <16 x float> *%a2_ptrt, <16 x i1> %mask) { -; CHECK-LABEL: test_x86_fmadd231_ps: -; CHECK: ## BB#0: -; CHECK-NEXT: vpmovsxbd %xmm2, %zmm2 -; CHECK-NEXT: vpandd {{.*}}(%rip){1to16}, %zmm2, %zmm2 -; CHECK-NEXT: vptestmd %zmm2, %zmm2, %k1 -; CHECK-NEXT: vfmadd231ps (%rdi), %zmm0, %zmm1 {%k1} -; CHECK-NEXT: vmovaps %zmm1, %zmm0 -; CHECK-NEXT: retq +; KNL-LABEL: test_x86_fmadd231_ps: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxbd %xmm2, %zmm2 +; KNL-NEXT: vpslld $31, %zmm2, %zmm2 +; KNL-NEXT: vptestmd %zmm2, %zmm2, %k1 +; KNL-NEXT: vfmadd231ps (%rdi), %zmm0, %zmm1 {%k1} +; KNL-NEXT: vmovaps %zmm1, %zmm0 +; KNL-NEXT: retq ; ; SKX-LABEL: test_x86_fmadd231_ps: ; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm2, %xmm2 ; SKX-NEXT: vpmovb2m %xmm2, %k1 ; SKX-NEXT: vfmadd231ps (%rdi), %zmm0, %zmm1 {%k1} ; SKX-NEXT: vmovaps %zmm1, %zmm0 @@ -163,17 +177,18 @@ define <16 x float> @test_x86_fmadd231_ps(<16 x float> %a0, <16 x float> %a1, <1 ;mask (b*a+c , b) define <16 x float> @test_x86_fmadd213_ps(<16 x float> %a0, <16 x float> %a1, <16 x float> *%a2_ptrt, <16 x i1> %mask) { -; CHECK-LABEL: test_x86_fmadd213_ps: -; CHECK: ## BB#0: -; CHECK-NEXT: vpmovsxbd %xmm2, %zmm2 -; CHECK-NEXT: vpandd {{.*}}(%rip){1to16}, %zmm2, %zmm2 -; CHECK-NEXT: vptestmd %zmm2, %zmm2, %k1 -; CHECK-NEXT: vfmadd213ps (%rdi), %zmm0, %zmm1 {%k1} -; CHECK-NEXT: vmovaps %zmm1, %zmm0 -; CHECK-NEXT: retq +; KNL-LABEL: test_x86_fmadd213_ps: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxbd %xmm2, %zmm2 +; KNL-NEXT: vpslld $31, %zmm2, %zmm2 +; KNL-NEXT: vptestmd %zmm2, %zmm2, %k1 +; KNL-NEXT: vfmadd213ps (%rdi), %zmm0, %zmm1 {%k1} +; KNL-NEXT: vmovaps %zmm1, %zmm0 +; KNL-NEXT: retq ; ; SKX-LABEL: test_x86_fmadd213_ps: ; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm2, %xmm2 ; SKX-NEXT: vpmovb2m %xmm2, %k1 ; SKX-NEXT: vfmadd213ps (%rdi), %zmm0, %zmm1 {%k1} ; SKX-NEXT: vmovaps %zmm1, %zmm0 diff --git a/test/CodeGen/X86/avx512-gather-scatter-intrin.ll b/test/CodeGen/X86/avx512-gather-scatter-intrin.ll index 3fca5a89a6a4..3bc67cceaab5 100644 --- a/test/CodeGen/X86/avx512-gather-scatter-intrin.ll +++ b/test/CodeGen/X86/avx512-gather-scatter-intrin.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s declare <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float>, i8*, <16 x i32>, i16, i32) @@ -14,7 +15,7 @@ define void @gather_mask_dps(<16 x i32> %ind, <16 x float> %src, i16 %mask, i8* ; CHECK-LABEL: gather_mask_dps: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovw %edi, %k1 -; CHECK-NEXT: kmovw %k1, %k2 +; CHECK-NEXT: kmovq %k1, %k2 ; CHECK-NEXT: vgatherdps (%rsi,%zmm0,4), %zmm1 {%k2} ; CHECK-NEXT: vpaddd {{.*}}(%rip), %zmm0, %zmm0 ; CHECK-NEXT: vscatterdps %zmm1, (%rdx,%zmm0,4) {%k1} @@ -29,7 +30,7 @@ define void @gather_mask_dpd(<8 x i32> %ind, <8 x double> %src, i8 %mask, i8* %b ; CHECK-LABEL: gather_mask_dpd: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %edi, %k1 -; CHECK-NEXT: kmovw %k1, %k2 +; CHECK-NEXT: kmovq %k1, %k2 ; CHECK-NEXT: vgatherdpd (%rsi,%ymm0,4), %zmm1 {%k2} ; CHECK-NEXT: vpaddd {{.*}}(%rip), %ymm0, %ymm0 ; CHECK-NEXT: vscatterdpd %zmm1, (%rdx,%ymm0,4) {%k1} @@ -44,7 +45,7 @@ define void @gather_mask_qps(<8 x i64> %ind, <8 x float> %src, i8 %mask, i8* %ba ; CHECK-LABEL: gather_mask_qps: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %edi, %k1 -; CHECK-NEXT: kmovw %k1, %k2 +; CHECK-NEXT: kmovq %k1, %k2 ; CHECK-NEXT: vgatherqps (%rsi,%zmm0,4), %ymm1 {%k2} ; CHECK-NEXT: vpaddq {{.*}}(%rip), %zmm0, %zmm0 ; CHECK-NEXT: vscatterqps %ymm1, (%rdx,%zmm0,4) {%k1} @@ -59,7 +60,7 @@ define void @gather_mask_qpd(<8 x i64> %ind, <8 x double> %src, i8 %mask, i8* %b ; CHECK-LABEL: gather_mask_qpd: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %edi, %k1 -; CHECK-NEXT: kmovw %k1, %k2 +; CHECK-NEXT: kmovq %k1, %k2 ; CHECK-NEXT: vgatherqpd (%rsi,%zmm0,4), %zmm1 {%k2} ; CHECK-NEXT: vpaddq {{.*}}(%rip), %zmm0, %zmm0 ; CHECK-NEXT: vscatterqpd %zmm1, (%rdx,%zmm0,4) {%k1} @@ -86,7 +87,7 @@ define void @gather_mask_dd(<16 x i32> %ind, <16 x i32> %src, i16 %mask, i8* %ba ; CHECK-LABEL: gather_mask_dd: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovw %edi, %k1 -; CHECK-NEXT: kmovw %k1, %k2 +; CHECK-NEXT: kmovq %k1, %k2 ; CHECK-NEXT: vpgatherdd (%rsi,%zmm0,4), %zmm1 {%k2} ; CHECK-NEXT: vpaddd {{.*}}(%rip), %zmm0, %zmm0 ; CHECK-NEXT: vpscatterdd %zmm1, (%rdx,%zmm0,4) {%k1} @@ -101,7 +102,7 @@ define void @gather_mask_qd(<8 x i64> %ind, <8 x i32> %src, i8 %mask, i8* %base, ; CHECK-LABEL: gather_mask_qd: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %edi, %k1 -; CHECK-NEXT: kmovw %k1, %k2 +; CHECK-NEXT: kmovq %k1, %k2 ; CHECK-NEXT: vpgatherqd (%rsi,%zmm0,4), %ymm1 {%k2} ; CHECK-NEXT: vpaddq {{.*}}(%rip), %zmm0, %zmm0 ; CHECK-NEXT: vpscatterqd %ymm1, (%rdx,%zmm0,4) {%k1} @@ -116,7 +117,7 @@ define void @gather_mask_qq(<8 x i64> %ind, <8 x i64> %src, i8 %mask, i8* %base, ; CHECK-LABEL: gather_mask_qq: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %edi, %k1 -; CHECK-NEXT: kmovw %k1, %k2 +; CHECK-NEXT: kmovq %k1, %k2 ; CHECK-NEXT: vpgatherqq (%rsi,%zmm0,4), %zmm1 {%k2} ; CHECK-NEXT: vpaddq {{.*}}(%rip), %zmm0, %zmm0 ; CHECK-NEXT: vpscatterqq %zmm1, (%rdx,%zmm0,4) {%k1} @@ -131,7 +132,7 @@ define void @gather_mask_dq(<8 x i32> %ind, <8 x i64> %src, i8 %mask, i8* %base, ; CHECK-LABEL: gather_mask_dq: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %edi, %k1 -; CHECK-NEXT: kmovw %k1, %k2 +; CHECK-NEXT: kmovq %k1, %k2 ; CHECK-NEXT: vpgatherdq (%rsi,%ymm0,4), %zmm1 {%k2} ; CHECK-NEXT: vpaddd {{.*}}(%rip), %ymm0, %ymm0 ; CHECK-NEXT: vpscatterdq %zmm1, (%rdx,%ymm0,4) {%k1} @@ -239,8 +240,8 @@ define void @scatter_mask_qps_execdomain(<8 x i64> %ind, <8 x float>* %src, i8 % define void @gather_qps(<8 x i64> %ind, <8 x float> %src, i8* %base, i8* %stbuf) { ; CHECK-LABEL: gather_qps: ; CHECK: ## BB#0: -; CHECK-NEXT: kxnorw %k1, %k1, %k1 -; CHECK-NEXT: kxnorw %k2, %k2, %k2 +; CHECK-NEXT: kxnorw %k0, %k0, %k1 +; CHECK-NEXT: kxnorw %k0, %k0, %k2 ; CHECK-NEXT: vgatherqps (%rdi,%zmm0,4), %ymm1 {%k2} ; CHECK-NEXT: vpaddq {{.*}}(%rip), %zmm0, %zmm0 ; CHECK-NEXT: vscatterqps %ymm1, (%rsi,%zmm0,4) {%k1} @@ -256,7 +257,7 @@ declare void @llvm.x86.avx512.scatterpf.qps.512(i8, <8 x i64>, i8* , i32, i32); define void @prefetch(<8 x i64> %ind, i8* %base) { ; CHECK-LABEL: prefetch: ; CHECK: ## BB#0: -; CHECK-NEXT: kxnorw %k1, %k1, %k1 +; CHECK-NEXT: kxnorw %k0, %k0, %k1 ; CHECK-NEXT: vgatherpf0qps (%rdi,%zmm0,4) {%k1} ; CHECK-NEXT: vgatherpf1qps (%rdi,%zmm0,4) {%k1} ; CHECK-NEXT: vscatterpf0qps (%rdi,%zmm0,2) {%k1} @@ -278,12 +279,12 @@ define <2 x double>@test_int_x86_avx512_gather3div2_df(<2 x double> %x0, i8* %x1 ; CHECK-NEXT: kmovb %esi, %k1 ; CHECK-NEXT: vmovaps %zmm0, %zmm2 ; CHECK-NEXT: vgatherqpd (%rdi,%xmm1,4), %xmm2 {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 -; CHECK-NEXT: vgatherqpd (%rdi,%xmm1,0), %xmm0 {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 +; CHECK-NEXT: vgatherqpd (%rdi,%xmm1,2), %xmm0 {%k1} ; CHECK-NEXT: vaddpd %xmm0, %xmm2, %xmm0 ; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.avx512.gather3div2.df(<2 x double> %x0, i8* %x1, <2 x i64> %x2, i8 %x3, i32 4) - %res1 = call <2 x double> @llvm.x86.avx512.gather3div2.df(<2 x double> %x0, i8* %x1, <2 x i64> %x2, i8 -1, i32 0) + %res1 = call <2 x double> @llvm.x86.avx512.gather3div2.df(<2 x double> %x0, i8* %x1, <2 x i64> %x2, i8 -1, i32 2) %res2 = fadd <2 x double> %res, %res1 ret <2 x double> %res2 } @@ -311,12 +312,12 @@ define <4 x double>@test_int_x86_avx512_gather3div4_df(<4 x double> %x0, i8* %x1 ; CHECK-NEXT: kmovb %esi, %k1 ; CHECK-NEXT: vmovaps %zmm0, %zmm2 ; CHECK-NEXT: vgatherqpd (%rdi,%ymm1,4), %ymm2 {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 -; CHECK-NEXT: vgatherqpd (%rdi,%ymm1,0), %ymm0 {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 +; CHECK-NEXT: vgatherqpd (%rdi,%ymm1,2), %ymm0 {%k1} ; CHECK-NEXT: vaddpd %ymm0, %ymm2, %ymm0 ; CHECK-NEXT: retq %res = call <4 x double> @llvm.x86.avx512.gather3div4.df(<4 x double> %x0, i8* %x1, <4 x i64> %x2, i8 %x3, i32 4) - %res1 = call <4 x double> @llvm.x86.avx512.gather3div4.df(<4 x double> %x0, i8* %x1, <4 x i64> %x2, i8 -1, i32 0) + %res1 = call <4 x double> @llvm.x86.avx512.gather3div4.df(<4 x double> %x0, i8* %x1, <4 x i64> %x2, i8 -1, i32 2) %res2 = fadd <4 x double> %res, %res1 ret <4 x double> %res2 } @@ -329,7 +330,7 @@ define <8 x i32>@test_int_x86_avx512_gather3div4_di(<4 x i64> %x0, i8* %x1, <4 x ; CHECK-NEXT: kmovb %esi, %k1 ; CHECK-NEXT: vmovaps %zmm0, %zmm2 ; CHECK-NEXT: vpgatherqq (%rdi,%ymm1,8), %ymm2 {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 +; CHECK-NEXT: kxnorw %k0, %k0, %k1 ; CHECK-NEXT: vpgatherqq (%rdi,%ymm1,8), %ymm0 {%k1} ; CHECK-NEXT: vpaddd %ymm0, %ymm2, %ymm0 ; CHECK-NEXT: retq @@ -347,12 +348,12 @@ define <4 x float>@test_int_x86_avx512_gather3div4_sf(<4 x float> %x0, i8* %x1, ; CHECK-NEXT: kmovb %esi, %k1 ; CHECK-NEXT: vmovaps %zmm0, %zmm2 ; CHECK-NEXT: vgatherqps (%rdi,%xmm1,4), %xmm2 {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 -; CHECK-NEXT: vgatherqps (%rdi,%xmm1,0), %xmm0 {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 +; CHECK-NEXT: vgatherqps (%rdi,%xmm1,2), %xmm0 {%k1} ; CHECK-NEXT: vaddps %xmm0, %xmm2, %xmm0 ; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.gather3div4.sf(<4 x float> %x0, i8* %x1, <2 x i64> %x2, i8 %x3, i32 4) - %res1 = call <4 x float> @llvm.x86.avx512.gather3div4.sf(<4 x float> %x0, i8* %x1, <2 x i64> %x2, i8 -1, i32 0) + %res1 = call <4 x float> @llvm.x86.avx512.gather3div4.sf(<4 x float> %x0, i8* %x1, <2 x i64> %x2, i8 -1, i32 2) %res2 = fadd <4 x float> %res, %res1 ret <4 x float> %res2 } @@ -363,7 +364,7 @@ define <4 x i32>@test_int_x86_avx512_gather3div4_si(<4 x i32> %x0, i8* %x1, <2 x ; CHECK-LABEL: test_int_x86_avx512_gather3div4_si: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: kxnorw %k2, %k2, %k2 +; CHECK-NEXT: kxnorw %k0, %k0, %k2 ; CHECK-NEXT: vmovaps %zmm0, %zmm2 ; CHECK-NEXT: vpgatherqd (%rdi,%xmm1,4), %xmm2 {%k2} ; CHECK-NEXT: vpgatherqd (%rdi,%xmm1,4), %xmm0 {%k1} @@ -383,12 +384,12 @@ define <4 x float>@test_int_x86_avx512_gather3div8_sf(<4 x float> %x0, i8* %x1, ; CHECK-NEXT: kmovb %esi, %k1 ; CHECK-NEXT: vmovaps %zmm0, %zmm2 ; CHECK-NEXT: vgatherqps (%rdi,%ymm1,4), %xmm2 {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 -; CHECK-NEXT: vgatherqps (%rdi,%ymm1,0), %xmm0 {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 +; CHECK-NEXT: vgatherqps (%rdi,%ymm1,2), %xmm0 {%k1} ; CHECK-NEXT: vaddps %xmm0, %xmm2, %xmm0 ; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.gather3div8.sf(<4 x float> %x0, i8* %x1, <4 x i64> %x2, i8 %x3, i32 4) - %res1 = call <4 x float> @llvm.x86.avx512.gather3div8.sf(<4 x float> %x0, i8* %x1, <4 x i64> %x2, i8 -1, i32 0) + %res1 = call <4 x float> @llvm.x86.avx512.gather3div8.sf(<4 x float> %x0, i8* %x1, <4 x i64> %x2, i8 -1, i32 2) %res2 = fadd <4 x float> %res, %res1 ret <4 x float> %res2 } @@ -400,7 +401,7 @@ define <4 x i32>@test_int_x86_avx512_gather3div8_si(<4 x i32> %x0, i8* %x1, <4 x ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 ; CHECK-NEXT: vmovaps %zmm0, %zmm2 -; CHECK-NEXT: kmovw %k1, %k2 +; CHECK-NEXT: kmovq %k1, %k2 ; CHECK-NEXT: vpgatherqd (%rdi,%ymm1,4), %xmm2 {%k2} ; CHECK-NEXT: vpgatherqd (%rdi,%ymm1,2), %xmm0 {%k1} ; CHECK-NEXT: vpaddd %xmm0, %xmm2, %xmm0 @@ -419,12 +420,12 @@ define <2 x double>@test_int_x86_avx512_gather3siv2_df(<2 x double> %x0, i8* %x1 ; CHECK-NEXT: kmovb %esi, %k1 ; CHECK-NEXT: vmovaps %zmm0, %zmm2 ; CHECK-NEXT: vgatherdpd (%rdi,%xmm1,4), %xmm2 {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 -; CHECK-NEXT: vgatherdpd (%rdi,%xmm1,0), %xmm0 {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 +; CHECK-NEXT: vgatherdpd (%rdi,%xmm1,2), %xmm0 {%k1} ; CHECK-NEXT: vaddpd %xmm0, %xmm2, %xmm0 ; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.avx512.gather3siv2.df(<2 x double> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 4) - %res1 = call <2 x double> @llvm.x86.avx512.gather3siv2.df(<2 x double> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 0) + %res1 = call <2 x double> @llvm.x86.avx512.gather3siv2.df(<2 x double> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 2) %res2 = fadd <2 x double> %res, %res1 ret <2 x double> %res2 } @@ -452,12 +453,12 @@ define <4 x double>@test_int_x86_avx512_gather3siv4_df(<4 x double> %x0, i8* %x1 ; CHECK-NEXT: kmovb %esi, %k1 ; CHECK-NEXT: vmovaps %zmm0, %zmm2 ; CHECK-NEXT: vgatherdpd (%rdi,%xmm1,4), %ymm2 {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 -; CHECK-NEXT: vgatherdpd (%rdi,%xmm1,0), %ymm0 {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 +; CHECK-NEXT: vgatherdpd (%rdi,%xmm1,2), %ymm0 {%k1} ; CHECK-NEXT: vaddpd %ymm0, %ymm2, %ymm0 ; CHECK-NEXT: retq %res = call <4 x double> @llvm.x86.avx512.gather3siv4.df(<4 x double> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 4) - %res1 = call <4 x double> @llvm.x86.avx512.gather3siv4.df(<4 x double> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 0) + %res1 = call <4 x double> @llvm.x86.avx512.gather3siv4.df(<4 x double> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 2) %res2 = fadd <4 x double> %res, %res1 ret <4 x double> %res2 } @@ -485,12 +486,12 @@ define <4 x float>@test_int_x86_avx512_gather3siv4_sf(<4 x float> %x0, i8* %x1, ; CHECK-NEXT: kmovb %esi, %k1 ; CHECK-NEXT: vmovaps %zmm0, %zmm2 ; CHECK-NEXT: vgatherdps (%rdi,%xmm1,4), %xmm2 {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 -; CHECK-NEXT: vgatherdps (%rdi,%xmm1,0), %xmm0 {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 +; CHECK-NEXT: vgatherdps (%rdi,%xmm1,2), %xmm0 {%k1} ; CHECK-NEXT: vaddps %xmm0, %xmm2, %xmm0 ; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.gather3siv4.sf(<4 x float> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 4) - %res1 = call <4 x float> @llvm.x86.avx512.gather3siv4.sf(<4 x float> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 0) + %res1 = call <4 x float> @llvm.x86.avx512.gather3siv4.sf(<4 x float> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 2) %res2 = fadd <4 x float> %res, %res1 ret <4 x float> %res2 } @@ -501,14 +502,14 @@ define <4 x i32>@test_int_x86_avx512_gather3siv4_si(<4 x i32> %x0, i8* %x1, <4 x ; CHECK-LABEL: test_int_x86_avx512_gather3siv4_si: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: kxnorw %k2, %k2, %k2 +; CHECK-NEXT: kxnorw %k0, %k0, %k2 ; CHECK-NEXT: vmovaps %zmm0, %zmm2 ; CHECK-NEXT: vpgatherdd (%rdi,%xmm1,4), %xmm2 {%k2} -; CHECK-NEXT: vpgatherdd (%rdi,%xmm1,0), %xmm0 {%k1} +; CHECK-NEXT: vpgatherdd (%rdi,%xmm1,2), %xmm0 {%k1} ; CHECK-NEXT: vpaddd %xmm0, %xmm2, %xmm0 ; CHECK-NEXT: retq %res = call <4 x i32> @llvm.x86.avx512.gather3siv4.si(<4 x i32> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 4) - %res1 = call <4 x i32> @llvm.x86.avx512.gather3siv4.si(<4 x i32> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 0) + %res1 = call <4 x i32> @llvm.x86.avx512.gather3siv4.si(<4 x i32> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 2) %res2 = add <4 x i32> %res, %res1 ret <4 x i32> %res2 } @@ -521,12 +522,12 @@ define <8 x float>@test_int_x86_avx512_gather3siv8_sf(<8 x float> %x0, i8* %x1, ; CHECK-NEXT: kmovb %esi, %k1 ; CHECK-NEXT: vmovaps %zmm0, %zmm2 ; CHECK-NEXT: vgatherdps (%rdi,%ymm1,4), %ymm2 {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 -; CHECK-NEXT: vgatherdps (%rdi,%ymm1,0), %ymm0 {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 +; CHECK-NEXT: vgatherdps (%rdi,%ymm1,2), %ymm0 {%k1} ; CHECK-NEXT: vaddps %ymm0, %ymm2, %ymm0 ; CHECK-NEXT: retq %res = call <8 x float> @llvm.x86.avx512.gather3siv8.sf(<8 x float> %x0, i8* %x1, <8 x i32> %x2, i8 %x3, i32 4) - %res1 = call <8 x float> @llvm.x86.avx512.gather3siv8.sf(<8 x float> %x0, i8* %x1, <8 x i32> %x2, i8 -1, i32 0) + %res1 = call <8 x float> @llvm.x86.avx512.gather3siv8.sf(<8 x float> %x0, i8* %x1, <8 x i32> %x2, i8 -1, i32 2) %res2 = fadd <8 x float> %res, %res1 ret <8 x float> %res2 } @@ -538,13 +539,13 @@ define <8 x i32>@test_int_x86_avx512_gather3siv8_si(<8 x i32> %x0, i8* %x1, <8 x ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 ; CHECK-NEXT: vmovaps %zmm0, %zmm2 -; CHECK-NEXT: kmovw %k1, %k2 +; CHECK-NEXT: kmovq %k1, %k2 ; CHECK-NEXT: vpgatherdd (%rdi,%ymm1,4), %ymm2 {%k2} -; CHECK-NEXT: vpgatherdd (%rdi,%ymm1,0), %ymm0 {%k1} +; CHECK-NEXT: vpgatherdd (%rdi,%ymm1,2), %ymm0 {%k1} ; CHECK-NEXT: vpaddd %ymm0, %ymm2, %ymm0 ; CHECK-NEXT: retq %res = call <8 x i32> @llvm.x86.avx512.gather3siv8.si(<8 x i32> %x0, i8* %x1, <8 x i32> %x2, i8 %x3, i32 4) - %res1 = call <8 x i32> @llvm.x86.avx512.gather3siv8.si(<8 x i32> %x0, i8* %x1, <8 x i32> %x2, i8 %x3, i32 0) + %res1 = call <8 x i32> @llvm.x86.avx512.gather3siv8.si(<8 x i32> %x0, i8* %x1, <8 x i32> %x2, i8 %x3, i32 2) %res2 = add <8 x i32> %res, %res1 ret <8 x i32> %res2 } @@ -555,11 +556,11 @@ define void@test_int_x86_avx512_scatterdiv2_df(i8* %x0, i8 %x1, <2 x i64> %x2, < ; CHECK-LABEL: test_int_x86_avx512_scatterdiv2_df: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: kxnorw %k2, %k2, %k2 -; CHECK-NEXT: vscatterqpd %xmm1, (%rdi,%xmm0,0) {%k2} +; CHECK-NEXT: kxnorw %k0, %k0, %k2 +; CHECK-NEXT: vscatterqpd %xmm1, (%rdi,%xmm0,2) {%k2} ; CHECK-NEXT: vscatterqpd %xmm1, (%rdi,%xmm0,4) {%k1} ; CHECK-NEXT: retq - call void @llvm.x86.avx512.scatterdiv2.df(i8* %x0, i8 -1, <2 x i64> %x2, <2 x double> %x3, i32 0) + call void @llvm.x86.avx512.scatterdiv2.df(i8* %x0, i8 -1, <2 x i64> %x2, <2 x double> %x3, i32 2) call void @llvm.x86.avx512.scatterdiv2.df(i8* %x0, i8 %x1, <2 x i64> %x2, <2 x double> %x3, i32 4) ret void } @@ -570,11 +571,11 @@ define void@test_int_x86_avx512_scatterdiv2_di(i8* %x0, i8 %x1, <2 x i64> %x2, < ; CHECK-LABEL: test_int_x86_avx512_scatterdiv2_di: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: vpscatterqq %xmm1, (%rdi,%xmm0,0) {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 +; CHECK-NEXT: vpscatterqq %xmm1, (%rdi,%xmm0,2) {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 ; CHECK-NEXT: vpscatterqq %xmm1, (%rdi,%xmm0,4) {%k1} ; CHECK-NEXT: retq - call void @llvm.x86.avx512.scatterdiv2.di(i8* %x0, i8 %x1, <2 x i64> %x2, <2 x i64> %x3, i32 0) + call void @llvm.x86.avx512.scatterdiv2.di(i8* %x0, i8 %x1, <2 x i64> %x2, <2 x i64> %x3, i32 2) call void @llvm.x86.avx512.scatterdiv2.di(i8* %x0, i8 -1, <2 x i64> %x2, <2 x i64> %x3, i32 4) ret void } @@ -585,11 +586,11 @@ define void@test_int_x86_avx512_scatterdiv4_df(i8* %x0, i8 %x1, <4 x i64> %x2, < ; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_df: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: vscatterqpd %ymm1, (%rdi,%ymm0,0) {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 +; CHECK-NEXT: vscatterqpd %ymm1, (%rdi,%ymm0,2) {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 ; CHECK-NEXT: vscatterqpd %ymm1, (%rdi,%ymm0,4) {%k1} ; CHECK-NEXT: retq - call void @llvm.x86.avx512.scatterdiv4.df(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x double> %x3, i32 0) + call void @llvm.x86.avx512.scatterdiv4.df(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x double> %x3, i32 2) call void @llvm.x86.avx512.scatterdiv4.df(i8* %x0, i8 -1, <4 x i64> %x2, <4 x double> %x3, i32 4) ret void } @@ -600,11 +601,11 @@ define void@test_int_x86_avx512_scatterdiv4_di(i8* %x0, i8 %x1, <4 x i64> %x2, < ; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_di: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: vpscatterqq %ymm1, (%rdi,%ymm0,0) {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 +; CHECK-NEXT: vpscatterqq %ymm1, (%rdi,%ymm0,2) {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 ; CHECK-NEXT: vpscatterqq %ymm1, (%rdi,%ymm0,4) {%k1} ; CHECK-NEXT: retq - call void @llvm.x86.avx512.scatterdiv4.di(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x i64> %x3, i32 0) + call void @llvm.x86.avx512.scatterdiv4.di(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x i64> %x3, i32 2) call void @llvm.x86.avx512.scatterdiv4.di(i8* %x0, i8 -1, <4 x i64> %x2, <4 x i64> %x3, i32 4) ret void } @@ -615,11 +616,11 @@ define void@test_int_x86_avx512_scatterdiv4_sf(i8* %x0, i8 %x1, <2 x i64> %x2, < ; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_sf: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: vscatterqps %xmm1, (%rdi,%xmm0,0) {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 +; CHECK-NEXT: vscatterqps %xmm1, (%rdi,%xmm0,2) {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 ; CHECK-NEXT: vscatterqps %xmm1, (%rdi,%xmm0,4) {%k1} ; CHECK-NEXT: retq - call void @llvm.x86.avx512.scatterdiv4.sf(i8* %x0, i8 %x1, <2 x i64> %x2, <4 x float> %x3, i32 0) + call void @llvm.x86.avx512.scatterdiv4.sf(i8* %x0, i8 %x1, <2 x i64> %x2, <4 x float> %x3, i32 2) call void @llvm.x86.avx512.scatterdiv4.sf(i8* %x0, i8 -1, <2 x i64> %x2, <4 x float> %x3, i32 4) ret void } @@ -630,11 +631,11 @@ define void@test_int_x86_avx512_scatterdiv4_si(i8* %x0, i8 %x1, <2 x i64> %x2, < ; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_si: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: kxnorw %k2, %k2, %k2 -; CHECK-NEXT: vpscatterqd %xmm1, (%rdi,%xmm0,0) {%k2} +; CHECK-NEXT: kxnorw %k0, %k0, %k2 +; CHECK-NEXT: vpscatterqd %xmm1, (%rdi,%xmm0,2) {%k2} ; CHECK-NEXT: vpscatterqd %xmm1, (%rdi,%xmm0,4) {%k1} ; CHECK-NEXT: retq - call void @llvm.x86.avx512.scatterdiv4.si(i8* %x0, i8 -1, <2 x i64> %x2, <4 x i32> %x3, i32 0) + call void @llvm.x86.avx512.scatterdiv4.si(i8* %x0, i8 -1, <2 x i64> %x2, <4 x i32> %x3, i32 2) call void @llvm.x86.avx512.scatterdiv4.si(i8* %x0, i8 %x1, <2 x i64> %x2, <4 x i32> %x3, i32 4) ret void } @@ -645,11 +646,11 @@ define void@test_int_x86_avx512_scatterdiv8_sf(i8* %x0, i8 %x1, <4 x i64> %x2, < ; CHECK-LABEL: test_int_x86_avx512_scatterdiv8_sf: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: vscatterqps %xmm1, (%rdi,%ymm0,0) {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 +; CHECK-NEXT: vscatterqps %xmm1, (%rdi,%ymm0,2) {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 ; CHECK-NEXT: vscatterqps %xmm1, (%rdi,%ymm0,4) {%k1} ; CHECK-NEXT: retq - call void @llvm.x86.avx512.scatterdiv8.sf(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x float> %x3, i32 0) + call void @llvm.x86.avx512.scatterdiv8.sf(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x float> %x3, i32 2) call void @llvm.x86.avx512.scatterdiv8.sf(i8* %x0, i8 -1, <4 x i64> %x2, <4 x float> %x3, i32 4) ret void } @@ -660,11 +661,11 @@ define void@test_int_x86_avx512_scatterdiv8_si(i8* %x0, i8 %x1, <4 x i64> %x2, < ; CHECK-LABEL: test_int_x86_avx512_scatterdiv8_si: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: vpscatterqd %xmm1, (%rdi,%ymm0,0) {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 +; CHECK-NEXT: vpscatterqd %xmm1, (%rdi,%ymm0,2) {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 ; CHECK-NEXT: vpscatterqd %xmm1, (%rdi,%ymm0,4) {%k1} ; CHECK-NEXT: retq - call void @llvm.x86.avx512.scatterdiv8.si(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x i32> %x3, i32 0) + call void @llvm.x86.avx512.scatterdiv8.si(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x i32> %x3, i32 2) call void @llvm.x86.avx512.scatterdiv8.si(i8* %x0, i8 -1, <4 x i64> %x2, <4 x i32> %x3, i32 4) ret void } @@ -675,11 +676,11 @@ define void@test_int_x86_avx512_scattersiv2_df(i8* %x0, i8 %x1, <4 x i32> %x2, < ; CHECK-LABEL: test_int_x86_avx512_scattersiv2_df: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: kxnorw %k2, %k2, %k2 -; CHECK-NEXT: vscatterdpd %xmm1, (%rdi,%xmm0,0) {%k2} +; CHECK-NEXT: kxnorw %k0, %k0, %k2 +; CHECK-NEXT: vscatterdpd %xmm1, (%rdi,%xmm0,2) {%k2} ; CHECK-NEXT: vscatterdpd %xmm1, (%rdi,%xmm0,4) {%k1} ; CHECK-NEXT: retq - call void @llvm.x86.avx512.scattersiv2.df(i8* %x0, i8 -1, <4 x i32> %x2, <2 x double> %x3, i32 0) + call void @llvm.x86.avx512.scattersiv2.df(i8* %x0, i8 -1, <4 x i32> %x2, <2 x double> %x3, i32 2) call void @llvm.x86.avx512.scattersiv2.df(i8* %x0, i8 %x1, <4 x i32> %x2, <2 x double> %x3, i32 4) ret void } @@ -690,11 +691,11 @@ define void@test_int_x86_avx512_scattersiv2_di(i8* %x0, i8 %x1, <4 x i32> %x2, < ; CHECK-LABEL: test_int_x86_avx512_scattersiv2_di: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: kxnorw %k2, %k2, %k2 -; CHECK-NEXT: vpscatterdq %xmm1, (%rdi,%xmm0,0) {%k2} +; CHECK-NEXT: kxnorw %k0, %k0, %k2 +; CHECK-NEXT: vpscatterdq %xmm1, (%rdi,%xmm0,2) {%k2} ; CHECK-NEXT: vpscatterdq %xmm1, (%rdi,%xmm0,4) {%k1} ; CHECK-NEXT: retq - call void @llvm.x86.avx512.scattersiv2.di(i8* %x0, i8 -1, <4 x i32> %x2, <2 x i64> %x3, i32 0) + call void @llvm.x86.avx512.scattersiv2.di(i8* %x0, i8 -1, <4 x i32> %x2, <2 x i64> %x3, i32 2) call void @llvm.x86.avx512.scattersiv2.di(i8* %x0, i8 %x1, <4 x i32> %x2, <2 x i64> %x3, i32 4) ret void } @@ -705,11 +706,11 @@ define void@test_int_x86_avx512_scattersiv4_df(i8* %x0, i8 %x1, <4 x i32> %x2, < ; CHECK-LABEL: test_int_x86_avx512_scattersiv4_df: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: vscatterdpd %ymm1, (%rdi,%xmm0,0) {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 +; CHECK-NEXT: vscatterdpd %ymm1, (%rdi,%xmm0,2) {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 ; CHECK-NEXT: vscatterdpd %ymm1, (%rdi,%xmm0,4) {%k1} ; CHECK-NEXT: retq - call void @llvm.x86.avx512.scattersiv4.df(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x double> %x3, i32 0) + call void @llvm.x86.avx512.scattersiv4.df(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x double> %x3, i32 2) call void @llvm.x86.avx512.scattersiv4.df(i8* %x0, i8 -1, <4 x i32> %x2, <4 x double> %x3, i32 4) ret void } @@ -720,11 +721,11 @@ define void@test_int_x86_avx512_scattersiv4_di(i8* %x0, i8 %x1, <4 x i32> %x2, < ; CHECK-LABEL: test_int_x86_avx512_scattersiv4_di: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: kxnorw %k2, %k2, %k2 -; CHECK-NEXT: vpscatterdq %ymm1, (%rdi,%xmm0,0) {%k2} +; CHECK-NEXT: kxnorw %k0, %k0, %k2 +; CHECK-NEXT: vpscatterdq %ymm1, (%rdi,%xmm0,2) {%k2} ; CHECK-NEXT: vpscatterdq %ymm1, (%rdi,%xmm0,4) {%k1} ; CHECK-NEXT: retq - call void @llvm.x86.avx512.scattersiv4.di(i8* %x0, i8 -1, <4 x i32> %x2, <4 x i64> %x3, i32 0) + call void @llvm.x86.avx512.scattersiv4.di(i8* %x0, i8 -1, <4 x i32> %x2, <4 x i64> %x3, i32 2) call void @llvm.x86.avx512.scattersiv4.di(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x i64> %x3, i32 4) ret void } @@ -735,11 +736,11 @@ define void@test_int_x86_avx512_scattersiv4_sf(i8* %x0, i8 %x1, <4 x i32> %x2, < ; CHECK-LABEL: test_int_x86_avx512_scattersiv4_sf: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: vscatterdps %xmm1, (%rdi,%xmm0,0) {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 +; CHECK-NEXT: vscatterdps %xmm1, (%rdi,%xmm0,2) {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 ; CHECK-NEXT: vscatterdps %xmm1, (%rdi,%xmm0,4) {%k1} ; CHECK-NEXT: retq - call void @llvm.x86.avx512.scattersiv4.sf(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x float> %x3, i32 0) + call void @llvm.x86.avx512.scattersiv4.sf(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x float> %x3, i32 2) call void @llvm.x86.avx512.scattersiv4.sf(i8* %x0, i8 -1, <4 x i32> %x2, <4 x float> %x3, i32 4) ret void } @@ -750,11 +751,11 @@ define void@test_int_x86_avx512_scattersiv4_si(i8* %x0, i8 %x1, <4 x i32> %x2, < ; CHECK-LABEL: test_int_x86_avx512_scattersiv4_si: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: vpscatterdd %xmm1, (%rdi,%xmm0,0) {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 +; CHECK-NEXT: vpscatterdd %xmm1, (%rdi,%xmm0,2) {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 ; CHECK-NEXT: vpscatterdd %xmm1, (%rdi,%xmm0,4) {%k1} ; CHECK-NEXT: retq - call void @llvm.x86.avx512.scattersiv4.si(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x i32> %x3, i32 0) + call void @llvm.x86.avx512.scattersiv4.si(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x i32> %x3, i32 2) call void @llvm.x86.avx512.scattersiv4.si(i8* %x0, i8 -1, <4 x i32> %x2, <4 x i32> %x3, i32 4) ret void } @@ -765,11 +766,11 @@ define void@test_int_x86_avx512_scattersiv8_sf(i8* %x0, i8 %x1, <8 x i32> %x2, < ; CHECK-LABEL: test_int_x86_avx512_scattersiv8_sf: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: vscatterdps %ymm1, (%rdi,%ymm0,0) {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 +; CHECK-NEXT: vscatterdps %ymm1, (%rdi,%ymm0,2) {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 ; CHECK-NEXT: vscatterdps %ymm1, (%rdi,%ymm0,4) {%k1} ; CHECK-NEXT: retq - call void @llvm.x86.avx512.scattersiv8.sf(i8* %x0, i8 %x1, <8 x i32> %x2, <8 x float> %x3, i32 0) + call void @llvm.x86.avx512.scattersiv8.sf(i8* %x0, i8 %x1, <8 x i32> %x2, <8 x float> %x3, i32 2) call void @llvm.x86.avx512.scattersiv8.sf(i8* %x0, i8 -1, <8 x i32> %x2, <8 x float> %x3, i32 4) ret void } @@ -780,11 +781,11 @@ define void@test_int_x86_avx512_scattersiv8_si(i8* %x0, i8 %x1, <8 x i32> %x2, < ; CHECK-LABEL: test_int_x86_avx512_scattersiv8_si: ; CHECK: ## BB#0: ; CHECK-NEXT: kmovb %esi, %k1 -; CHECK-NEXT: vpscatterdd %ymm1, (%rdi,%ymm0,0) {%k1} -; CHECK-NEXT: kxnorw %k1, %k1, %k1 +; CHECK-NEXT: vpscatterdd %ymm1, (%rdi,%ymm0,2) {%k1} +; CHECK-NEXT: kxnorw %k0, %k0, %k1 ; CHECK-NEXT: vpscatterdd %ymm1, (%rdi,%ymm0,4) {%k1} ; CHECK-NEXT: retq - call void @llvm.x86.avx512.scattersiv8.si(i8* %x0, i8 %x1, <8 x i32> %x2, <8 x i32> %x3, i32 0) + call void @llvm.x86.avx512.scattersiv8.si(i8* %x0, i8 %x1, <8 x i32> %x2, <8 x i32> %x3, i32 2) call void @llvm.x86.avx512.scattersiv8.si(i8* %x0, i8 -1, <8 x i32> %x2, <8 x i32> %x3, i32 4) ret void } diff --git a/test/CodeGen/X86/avx512-insert-extract.ll b/test/CodeGen/X86/avx512-insert-extract.ll index 6f985f0bf3a7..41ec62c7e047 100644 --- a/test/CodeGen/X86/avx512-insert-extract.ll +++ b/test/CodeGen/X86/avx512-insert-extract.ll @@ -12,14 +12,24 @@ define <16 x float> @test1(<16 x float> %x, float* %br, float %y) nounwind { ret <16 x float> %rrr3 } -;CHECK-LABEL: test2: -;KNL: vinsertf32x4 $0 -;SKX: vinsertf64x2 $0 -;CHECK: vextractf32x4 $3 -;KNL: vinsertf32x4 $3 -;SKX: vinsertf64x2 $3 -;CHECK: ret define <8 x double> @test2(<8 x double> %x, double* %br, double %y) nounwind { +; KNL-LABEL: test2: +; KNL: ## BB#0: +; KNL-NEXT: vmovhpd (%rdi), %xmm0, %xmm2 +; KNL-NEXT: vinsertf32x4 $0, %xmm2, %zmm0, %zmm0 +; KNL-NEXT: vextractf32x4 $3, %zmm0, %xmm2 +; KNL-NEXT: vmovsd %xmm1, %xmm2, %xmm1 +; KNL-NEXT: vinsertf32x4 $3, %xmm1, %zmm0, %zmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test2: +; SKX: ## BB#0: +; SKX-NEXT: vmovhpd (%rdi), %xmm0, %xmm2 +; SKX-NEXT: vinsertf64x2 $0, %xmm2, %zmm0, %zmm0 +; SKX-NEXT: vextractf64x2 $3, %zmm0, %xmm2 +; SKX-NEXT: vmovsd %xmm1, %xmm2, %xmm1 +; SKX-NEXT: vinsertf64x2 $3, %xmm1, %zmm0, %zmm0 +; SKX-NEXT: retq %rrr = load double, double* %br %rrr2 = insertelement <8 x double> %x, double %rrr, i32 1 %rrr3 = insertelement <8 x double> %rrr2, double %y, i32 6 @@ -36,12 +46,22 @@ define <16 x float> @test3(<16 x float> %x) nounwind { ret <16 x float> %rrr2 } -;CHECK-LABEL: test4: -;CHECK: vextracti32x4 $2 -;KNL: vinserti32x4 $0 -;SKX: vinserti64x2 $0 -;CHECK: ret define <8 x i64> @test4(<8 x i64> %x) nounwind { +; KNL-LABEL: test4: +; KNL: ## BB#0: +; KNL-NEXT: vextracti32x4 $2, %zmm0, %xmm1 +; KNL-NEXT: vmovq %xmm1, %rax +; KNL-NEXT: vpinsrq $1, %rax, %xmm0, %xmm1 +; KNL-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test4: +; SKX: ## BB#0: +; SKX-NEXT: vextracti64x2 $2, %zmm0, %xmm1 +; SKX-NEXT: vmovq %xmm1, %rax +; SKX-NEXT: vpinsrq $1, %rax, %xmm0, %xmm1 +; SKX-NEXT: vinserti64x2 $0, %xmm1, %zmm0, %zmm0 +; SKX-NEXT: retq %eee = extractelement <8 x i64> %x, i32 4 %rrr2 = insertelement <8 x i64> %x, i64 %eee, i32 1 ret <8 x i64> %rrr2 @@ -142,7 +162,7 @@ define i64 @test12(<16 x i64>%a, <16 x i64>%b, i64 %a1, i64 %b1) { ;CHECK: andl $1, %eax ;CHECK: kmovw %eax, %k0 ;CHECK: movw $-4 -;CHECK: korw +;CHECK: korw define i16 @test13(i32 %a, i32 %b) { %cmp_res = icmp ult i32 %a, %b %maskv = insertelement <16 x i1> , i1 %cmp_res, i32 0 @@ -211,3 +231,476 @@ define i8 @test17(i1 *%addr, i8 %a) { ret i8 %x2 } +define i64 @extract_v8i64(<8 x i64> %x, i64* %dst) { +; SKX-LABEL: extract_v8i64: +; SKX: ## BB#0: +; SKX-NEXT: vpextrq $1, %xmm0, %rax +; SKX-NEXT: vextracti64x2 $1, %zmm0, %xmm0 +; SKX-NEXT: vpextrq $1, %xmm0, (%rdi) +; SKX-NEXT: retq + %r1 = extractelement <8 x i64> %x, i32 1 + %r2 = extractelement <8 x i64> %x, i32 3 + store i64 %r2, i64* %dst, align 1 + ret i64 %r1 +} + +define i64 @extract_v4i64(<4 x i64> %x, i64* %dst) { +; SKX-LABEL: extract_v4i64: +; SKX: ## BB#0: +; SKX-NEXT: vpextrq $1, %xmm0, %rax +; SKX-NEXT: vextracti128 $1, %ymm0, %xmm0 +; SKX-NEXT: vpextrq $1, %xmm0, (%rdi) +; SKX-NEXT: retq + %r1 = extractelement <4 x i64> %x, i32 1 + %r2 = extractelement <4 x i64> %x, i32 3 + store i64 %r2, i64* %dst, align 1 + ret i64 %r1 +} + +define i64 @extract_v2i64(<2 x i64> %x, i64* %dst) { +; SKX-LABEL: extract_v2i64: +; SKX: ## BB#0: +; SKX-NEXT: vmovq %xmm0, %rax +; SKX-NEXT: vpextrq $1, %xmm0, (%rdi) +; SKX-NEXT: retq + %r1 = extractelement <2 x i64> %x, i32 0 + %r2 = extractelement <2 x i64> %x, i32 1 + store i64 %r2, i64* %dst, align 1 + ret i64 %r1 +} + +define i32 @extract_v16i32(<16 x i32> %x, i32* %dst) { +; SKX-LABEL: extract_v16i32: +; SKX: ## BB#0: +; SKX-NEXT: vpextrd $1, %xmm0, %eax +; SKX-NEXT: vextracti32x4 $1, %zmm0, %xmm0 +; SKX-NEXT: vpextrd $1, %xmm0, (%rdi) +; SKX-NEXT: retq + %r1 = extractelement <16 x i32> %x, i32 1 + %r2 = extractelement <16 x i32> %x, i32 5 + store i32 %r2, i32* %dst, align 1 + ret i32 %r1 +} + +define i32 @extract_v8i32(<8 x i32> %x, i32* %dst) { +; SKX-LABEL: extract_v8i32: +; SKX: ## BB#0: +; SKX-NEXT: vpextrd $1, %xmm0, %eax +; SKX-NEXT: vextracti128 $1, %ymm0, %xmm0 +; SKX-NEXT: vpextrd $1, %xmm0, (%rdi) +; SKX-NEXT: retq + %r1 = extractelement <8 x i32> %x, i32 1 + %r2 = extractelement <8 x i32> %x, i32 5 + store i32 %r2, i32* %dst, align 1 + ret i32 %r1 +} + +define i32 @extract_v4i32(<4 x i32> %x, i32* %dst) { +; SKX-LABEL: extract_v4i32: +; SKX: ## BB#0: +; SKX-NEXT: vpextrd $1, %xmm0, %eax +; SKX-NEXT: vpextrd $3, %xmm0, (%rdi) +; SKX-NEXT: retq + %r1 = extractelement <4 x i32> %x, i32 1 + %r2 = extractelement <4 x i32> %x, i32 3 + store i32 %r2, i32* %dst, align 1 + ret i32 %r1 +} + +define i16 @extract_v32i16(<32 x i16> %x, i16* %dst) { +; SKX-LABEL: extract_v32i16: +; SKX: ## BB#0: +; SKX-NEXT: vpextrw $1, %xmm0, %eax +; SKX-NEXT: vextracti32x4 $1, %zmm0, %xmm0 +; SKX-NEXT: vpextrw $1, %xmm0, (%rdi) +; SKX-NEXT: retq + %r1 = extractelement <32 x i16> %x, i32 1 + %r2 = extractelement <32 x i16> %x, i32 9 + store i16 %r2, i16* %dst, align 1 + ret i16 %r1 +} + +define i16 @extract_v16i16(<16 x i16> %x, i16* %dst) { +; SKX-LABEL: extract_v16i16: +; SKX: ## BB#0: +; SKX-NEXT: vpextrw $1, %xmm0, %eax +; SKX-NEXT: vextracti128 $1, %ymm0, %xmm0 +; SKX-NEXT: vpextrw $1, %xmm0, (%rdi) +; SKX-NEXT: retq + %r1 = extractelement <16 x i16> %x, i32 1 + %r2 = extractelement <16 x i16> %x, i32 9 + store i16 %r2, i16* %dst, align 1 + ret i16 %r1 +} + +define i16 @extract_v8i16(<8 x i16> %x, i16* %dst) { +; SKX-LABEL: extract_v8i16: +; SKX: ## BB#0: +; SKX-NEXT: vpextrw $1, %xmm0, %eax +; SKX-NEXT: vpextrw $3, %xmm0, (%rdi) +; SKX-NEXT: retq + %r1 = extractelement <8 x i16> %x, i32 1 + %r2 = extractelement <8 x i16> %x, i32 3 + store i16 %r2, i16* %dst, align 1 + ret i16 %r1 +} + +define i8 @extract_v64i8(<64 x i8> %x, i8* %dst) { +; SKX-LABEL: extract_v64i8: +; SKX: ## BB#0: +; SKX-NEXT: vpextrb $1, %xmm0, %eax +; SKX-NEXT: vextracti32x4 $1, %zmm0, %xmm0 +; SKX-NEXT: vpextrb $1, %xmm0, (%rdi) +; SKX-NEXT: retq + %r1 = extractelement <64 x i8> %x, i32 1 + %r2 = extractelement <64 x i8> %x, i32 17 + store i8 %r2, i8* %dst, align 1 + ret i8 %r1 +} + +define i8 @extract_v32i8(<32 x i8> %x, i8* %dst) { +; SKX-LABEL: extract_v32i8: +; SKX: ## BB#0: +; SKX-NEXT: vpextrb $1, %xmm0, %eax +; SKX-NEXT: vextracti128 $1, %ymm0, %xmm0 +; SKX-NEXT: vpextrb $1, %xmm0, (%rdi) +; SKX-NEXT: retq + %r1 = extractelement <32 x i8> %x, i32 1 + %r2 = extractelement <32 x i8> %x, i32 17 + store i8 %r2, i8* %dst, align 1 + ret i8 %r1 +} + +define i8 @extract_v16i8(<16 x i8> %x, i8* %dst) { +; SKX-LABEL: extract_v16i8: +; SKX: ## BB#0: +; SKX-NEXT: vpextrb $1, %xmm0, %eax +; SKX-NEXT: vpextrb $3, %xmm0, (%rdi) +; SKX-NEXT: retq + %r1 = extractelement <16 x i8> %x, i32 1 + %r2 = extractelement <16 x i8> %x, i32 3 + store i8 %r2, i8* %dst, align 1 + ret i8 %r1 +} + +define <8 x i64> @insert_v8i64(<8 x i64> %x, i64 %y , i64* %ptr) { +; SKX-LABEL: insert_v8i64: +; SKX: ## BB#0: +; SKX-NEXT: vpinsrq $1, (%rsi), %xmm0, %xmm1 +; SKX-NEXT: vinserti64x2 $0, %xmm1, %zmm0, %zmm0 +; SKX-NEXT: vextracti64x2 $1, %zmm0, %xmm1 +; SKX-NEXT: vpinsrq $1, %rdi, %xmm1, %xmm1 +; SKX-NEXT: vinserti64x2 $1, %xmm1, %zmm0, %zmm0 +; SKX-NEXT: retq + %val = load i64, i64* %ptr + %r1 = insertelement <8 x i64> %x, i64 %val, i32 1 + %r2 = insertelement <8 x i64> %r1, i64 %y, i32 3 + ret <8 x i64> %r2 +} + +define <4 x i64> @insert_v4i64(<4 x i64> %x, i64 %y , i64* %ptr) { +; SKX-LABEL: insert_v4i64: +; SKX: ## BB#0: +; SKX-NEXT: vpinsrq $1, (%rsi), %xmm0, %xmm1 +; SKX-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; SKX-NEXT: vextracti128 $1, %ymm0, %xmm1 +; SKX-NEXT: vpinsrq $1, %rdi, %xmm1, %xmm1 +; SKX-NEXT: vinserti64x2 $1, %xmm1, %ymm0, %ymm0 +; SKX-NEXT: retq + %val = load i64, i64* %ptr + %r1 = insertelement <4 x i64> %x, i64 %val, i32 1 + %r2 = insertelement <4 x i64> %r1, i64 %y, i32 3 + ret <4 x i64> %r2 +} + +define <2 x i64> @insert_v2i64(<2 x i64> %x, i64 %y , i64* %ptr) { +; SKX-LABEL: insert_v2i64: +; SKX: ## BB#0: +; SKX-NEXT: vpinsrq $1, (%rsi), %xmm0, %xmm0 +; SKX-NEXT: vpinsrq $3, %rdi, %xmm0, %xmm0 +; SKX-NEXT: retq + %val = load i64, i64* %ptr + %r1 = insertelement <2 x i64> %x, i64 %val, i32 1 + %r2 = insertelement <2 x i64> %r1, i64 %y, i32 3 + ret <2 x i64> %r2 +} + +define <16 x i32> @insert_v16i32(<16 x i32> %x, i32 %y, i32* %ptr) { +; SKX-LABEL: insert_v16i32: +; SKX: ## BB#0: +; SKX-NEXT: vpinsrd $1, (%rsi), %xmm0, %xmm1 +; SKX-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0 +; SKX-NEXT: vextracti32x4 $1, %zmm0, %xmm1 +; SKX-NEXT: vpinsrd $1, %edi, %xmm1, %xmm1 +; SKX-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0 +; SKX-NEXT: retq + %val = load i32, i32* %ptr + %r1 = insertelement <16 x i32> %x, i32 %val, i32 1 + %r2 = insertelement <16 x i32> %r1, i32 %y, i32 5 + ret <16 x i32> %r2 +} + +define <8 x i32> @insert_v8i32(<8 x i32> %x, i32 %y, i32* %ptr) { +; KNL-LABEL: insert_v8i32: +; KNL: ## BB#0: +; KNL-NEXT: vpinsrd $1, (%rsi), %xmm0, %xmm1 +; KNL-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; KNL-NEXT: vextracti128 $1, %ymm0, %xmm1 +; KNL-NEXT: vpinsrd $1, %edi, %xmm1, %xmm1 +; KNL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: insert_v8i32: +; SKX: ## BB#0: +; SKX-NEXT: vpinsrd $1, (%rsi), %xmm0, %xmm1 +; SKX-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; SKX-NEXT: vextracti128 $1, %ymm0, %xmm1 +; SKX-NEXT: vpinsrd $1, %edi, %xmm1, %xmm1 +; SKX-NEXT: vinserti32x4 $1, %xmm1, %ymm0, %ymm0 +; SKX-NEXT: retq + %val = load i32, i32* %ptr + %r1 = insertelement <8 x i32> %x, i32 %val, i32 1 + %r2 = insertelement <8 x i32> %r1, i32 %y, i32 5 + ret <8 x i32> %r2 +} + +define <4 x i32> @insert_v4i32(<4 x i32> %x, i32 %y, i32* %ptr) { +; KNL-LABEL: insert_v4i32: +; KNL: ## BB#0: +; KNL-NEXT: vpinsrd $1, (%rsi), %xmm0, %xmm0 +; KNL-NEXT: vpinsrd $3, %edi, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: insert_v4i32: +; SKX: ## BB#0: +; SKX-NEXT: vpinsrd $1, (%rsi), %xmm0, %xmm0 +; SKX-NEXT: vpinsrd $3, %edi, %xmm0, %xmm0 +; SKX-NEXT: retq + %val = load i32, i32* %ptr + %r1 = insertelement <4 x i32> %x, i32 %val, i32 1 + %r2 = insertelement <4 x i32> %r1, i32 %y, i32 3 + ret <4 x i32> %r2 +} + +define <32 x i16> @insert_v32i16(<32 x i16> %x, i16 %y, i16* %ptr) { +; KNL-LABEL: insert_v32i16: +; KNL: ## BB#0: +; KNL-NEXT: vpinsrw $1, (%rsi), %xmm0, %xmm2 +; KNL-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] +; KNL-NEXT: vextracti128 $1, %ymm0, %xmm2 +; KNL-NEXT: vpinsrw $1, %edi, %xmm2, %xmm2 +; KNL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: insert_v32i16: +; SKX: ## BB#0: +; SKX-NEXT: vpinsrw $1, (%rsi), %xmm0, %xmm1 +; SKX-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0 +; SKX-NEXT: vextracti32x4 $1, %zmm0, %xmm1 +; SKX-NEXT: vpinsrw $1, %edi, %xmm1, %xmm1 +; SKX-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0 +; SKX-NEXT: retq + %val = load i16, i16* %ptr + %r1 = insertelement <32 x i16> %x, i16 %val, i32 1 + %r2 = insertelement <32 x i16> %r1, i16 %y, i32 9 + ret <32 x i16> %r2 +} + +define <16 x i16> @insert_v16i16(<16 x i16> %x, i16 %y, i16* %ptr) { +; KNL-LABEL: insert_v16i16: +; KNL: ## BB#0: +; KNL-NEXT: vpinsrw $1, (%rsi), %xmm0, %xmm1 +; KNL-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; KNL-NEXT: vextracti128 $1, %ymm0, %xmm1 +; KNL-NEXT: vpinsrw $1, %edi, %xmm1, %xmm1 +; KNL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: insert_v16i16: +; SKX: ## BB#0: +; SKX-NEXT: vpinsrw $1, (%rsi), %xmm0, %xmm1 +; SKX-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; SKX-NEXT: vextracti128 $1, %ymm0, %xmm1 +; SKX-NEXT: vpinsrw $1, %edi, %xmm1, %xmm1 +; SKX-NEXT: vinserti32x4 $1, %xmm1, %ymm0, %ymm0 +; SKX-NEXT: retq + %val = load i16, i16* %ptr + %r1 = insertelement <16 x i16> %x, i16 %val, i32 1 + %r2 = insertelement <16 x i16> %r1, i16 %y, i32 9 + ret <16 x i16> %r2 +} + +define <8 x i16> @insert_v8i16(<8 x i16> %x, i16 %y, i16* %ptr) { +; KNL-LABEL: insert_v8i16: +; KNL: ## BB#0: +; KNL-NEXT: vpinsrw $1, (%rsi), %xmm0, %xmm0 +; KNL-NEXT: vpinsrw $5, %edi, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: insert_v8i16: +; SKX: ## BB#0: +; SKX-NEXT: vpinsrw $1, (%rsi), %xmm0, %xmm0 +; SKX-NEXT: vpinsrw $5, %edi, %xmm0, %xmm0 +; SKX-NEXT: retq + %val = load i16, i16* %ptr + %r1 = insertelement <8 x i16> %x, i16 %val, i32 1 + %r2 = insertelement <8 x i16> %r1, i16 %y, i32 5 + ret <8 x i16> %r2 +} + +define <64 x i8> @insert_v64i8(<64 x i8> %x, i8 %y, i8* %ptr) { +; KNL-LABEL: insert_v64i8: +; KNL: ## BB#0: +; KNL-NEXT: vpinsrb $1, (%rsi), %xmm0, %xmm2 +; KNL-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] +; KNL-NEXT: vextracti128 $1, %ymm1, %xmm2 +; KNL-NEXT: vpinsrb $2, %edi, %xmm2, %xmm2 +; KNL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; KNL-NEXT: retq +; +; SKX-LABEL: insert_v64i8: +; SKX: ## BB#0: +; SKX-NEXT: vpinsrb $1, (%rsi), %xmm0, %xmm1 +; SKX-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0 +; SKX-NEXT: vextracti32x4 $3, %zmm0, %xmm1 +; SKX-NEXT: vpinsrb $2, %edi, %xmm1, %xmm1 +; SKX-NEXT: vinserti32x4 $3, %xmm1, %zmm0, %zmm0 +; SKX-NEXT: retq + %val = load i8, i8* %ptr + %r1 = insertelement <64 x i8> %x, i8 %val, i32 1 + %r2 = insertelement <64 x i8> %r1, i8 %y, i32 50 + ret <64 x i8> %r2 +} + +define <32 x i8> @insert_v32i8(<32 x i8> %x, i8 %y, i8* %ptr) { +; SKX-LABEL: insert_v32i8: +; SKX: ## BB#0: +; SKX-NEXT: vpinsrb $1, (%rsi), %xmm0, %xmm1 +; SKX-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; SKX-NEXT: vextracti128 $1, %ymm0, %xmm1 +; SKX-NEXT: vpinsrb $1, %edi, %xmm1, %xmm1 +; SKX-NEXT: vinserti32x4 $1, %xmm1, %ymm0, %ymm0 +; SKX-NEXT: retq + %val = load i8, i8* %ptr + %r1 = insertelement <32 x i8> %x, i8 %val, i32 1 + %r2 = insertelement <32 x i8> %r1, i8 %y, i32 17 + ret <32 x i8> %r2 +} + +define <16 x i8> @insert_v16i8(<16 x i8> %x, i8 %y, i8* %ptr) { +; KNL-LABEL: insert_v16i8: +; KNL: ## BB#0: +; KNL-NEXT: vpinsrb $3, (%rsi), %xmm0, %xmm0 +; KNL-NEXT: vpinsrb $10, %edi, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: insert_v16i8: +; SKX: ## BB#0: +; SKX-NEXT: vpinsrb $3, (%rsi), %xmm0, %xmm0 +; SKX-NEXT: vpinsrb $10, %edi, %xmm0, %xmm0 +; SKX-NEXT: retq + %val = load i8, i8* %ptr + %r1 = insertelement <16 x i8> %x, i8 %val, i32 3 + %r2 = insertelement <16 x i8> %r1, i8 %y, i32 10 + ret <16 x i8> %r2 +} + +define <8 x i64> @test_insert_128_v8i64(<8 x i64> %x, i64 %y) { +; KNL-LABEL: test_insert_128_v8i64: +; KNL: ## BB#0: +; KNL-NEXT: vpinsrq $1, %rdi, %xmm0, %xmm1 +; KNL-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test_insert_128_v8i64: +; SKX: ## BB#0: +; SKX-NEXT: vpinsrq $1, %rdi, %xmm0, %xmm1 +; SKX-NEXT: vinserti64x2 $0, %xmm1, %zmm0, %zmm0 +; SKX-NEXT: retq + %r = insertelement <8 x i64> %x, i64 %y, i32 1 + ret <8 x i64> %r +} + +define <16 x i32> @test_insert_128_v16i32(<16 x i32> %x, i32 %y) { +; KNL-LABEL: test_insert_128_v16i32: +; KNL: ## BB#0: +; KNL-NEXT: vpinsrd $1, %edi, %xmm0, %xmm1 +; KNL-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test_insert_128_v16i32: +; SKX: ## BB#0: +; SKX-NEXT: vpinsrd $1, %edi, %xmm0, %xmm1 +; SKX-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0 +; SKX-NEXT: retq + %r = insertelement <16 x i32> %x, i32 %y, i32 1 + ret <16 x i32> %r +} + +define <8 x double> @test_insert_128_v8f64(<8 x double> %x, double %y) { +; KNL-LABEL: test_insert_128_v8f64: +; KNL: ## BB#0: +; KNL-NEXT: vunpcklpd {{.*#+}} xmm1 = xmm0[0],xmm1[0] +; KNL-NEXT: vinsertf32x4 $0, %xmm1, %zmm0, %zmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test_insert_128_v8f64: +; SKX: ## BB#0: +; SKX-NEXT: vunpcklpd %xmm1, %xmm0, %xmm1 +; SKX-NEXT: vinsertf64x2 $0, %xmm1, %zmm0, %zmm0 +; SKX-NEXT: retq + %r = insertelement <8 x double> %x, double %y, i32 1 + ret <8 x double> %r +} + +define <16 x float> @test_insert_128_v16f32(<16 x float> %x, float %y) { +; KNL-LABEL: test_insert_128_v16f32: +; KNL: ## BB#0: +; KNL-NEXT: vinsertps $16, %xmm1, %xmm0, %xmm1 +; KNL-NEXT: vinsertf32x4 $0, %xmm1, %zmm0, %zmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test_insert_128_v16f32: +; SKX: ## BB#0: +; SKX-NEXT: vinsertps $16, %xmm1, %xmm0, %xmm1 +; SKX-NEXT: vinsertf32x4 $0, %xmm1, %zmm0, %zmm0 +; SKX-NEXT: retq + %r = insertelement <16 x float> %x, float %y, i32 1 + ret <16 x float> %r +} + +define <16 x i16> @test_insert_128_v16i16(<16 x i16> %x, i16 %y) { +; KNL-LABEL: test_insert_128_v16i16: +; KNL: ## BB#0: +; KNL-NEXT: vextracti128 $1, %ymm0, %xmm1 +; KNL-NEXT: vpinsrw $2, %edi, %xmm1, %xmm1 +; KNL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test_insert_128_v16i16: +; SKX: ## BB#0: +; SKX-NEXT: vextracti128 $1, %ymm0, %xmm1 +; SKX-NEXT: vpinsrw $2, %edi, %xmm1, %xmm1 +; SKX-NEXT: vinserti32x4 $1, %xmm1, %ymm0, %ymm0 +; SKX-NEXT: retq + %r = insertelement <16 x i16> %x, i16 %y, i32 10 + ret <16 x i16> %r +} + +define <32 x i8> @test_insert_128_v32i8(<32 x i8> %x, i8 %y) { +; KNL-LABEL: test_insert_128_v32i8: +; KNL: ## BB#0: +; KNL-NEXT: vextracti128 $1, %ymm0, %xmm1 +; KNL-NEXT: vpinsrb $4, %edi, %xmm1, %xmm1 +; KNL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test_insert_128_v32i8: +; SKX: ## BB#0: +; SKX-NEXT: vextracti128 $1, %ymm0, %xmm1 +; SKX-NEXT: vpinsrb $4, %edi, %xmm1, %xmm1 +; SKX-NEXT: vinserti32x4 $1, %xmm1, %ymm0, %ymm0 +; SKX-NEXT: retq + %r = insertelement <32 x i8> %x, i8 %y, i32 20 + ret <32 x i8> %r +} diff --git a/test/CodeGen/X86/avx512-intrinsics.ll b/test/CodeGen/X86/avx512-intrinsics.ll index 7642cd4e6c5c..764e13638485 100644 --- a/test/CodeGen/X86/avx512-intrinsics.ll +++ b/test/CodeGen/X86/avx512-intrinsics.ll @@ -1,60 +1,94 @@ -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl --show-mc-encoding| FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s declare i32 @llvm.x86.avx512.kortestz.w(i16, i16) nounwind readnone -; CHECK-LABEL: test_kortestz -; CHECK: kortestw -; CHECK: sete define i32 @test_kortestz(i16 %a0, i16 %a1) { +; CHECK-LABEL: test_kortestz: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k0 +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: kortestw %k0, %k1 +; CHECK-NEXT: sete %al +; CHECK-NEXT: kmovw %eax, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: andl $1, %eax +; CHECK-NEXT: retq %res = call i32 @llvm.x86.avx512.kortestz.w(i16 %a0, i16 %a1) ret i32 %res } declare i32 @llvm.x86.avx512.kortestc.w(i16, i16) nounwind readnone -; CHECK-LABEL: test_kortestc -; CHECK: kortestw -; CHECK: sbbl define i32 @test_kortestc(i16 %a0, i16 %a1) { +; CHECK-LABEL: test_kortestc: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k0 +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: kortestw %k0, %k1 +; CHECK-NEXT: sbbl %eax, %eax +; CHECK-NEXT: andl $1, %eax +; CHECK-NEXT: retq %res = call i32 @llvm.x86.avx512.kortestc.w(i16 %a0, i16 %a1) ret i32 %res } declare i16 @llvm.x86.avx512.kand.w(i16, i16) nounwind readnone -; CHECK-LABEL: test_kand -; CHECK: kandw -; CHECK: kandw define i16 @test_kand(i16 %a0, i16 %a1) { +; CHECK-LABEL: test_kand: +; CHECK: ## BB#0: +; CHECK-NEXT: movw $8, %ax +; CHECK-NEXT: kmovw %eax, %k0 +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: kandw %k0, %k1, %k0 +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: kandw %k1, %k0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq %t1 = call i16 @llvm.x86.avx512.kand.w(i16 %a0, i16 8) %t2 = call i16 @llvm.x86.avx512.kand.w(i16 %t1, i16 %a1) ret i16 %t2 } declare i16 @llvm.x86.avx512.knot.w(i16) nounwind readnone -; CHECK-LABEL: test_knot -; CHECK: knotw define i16 @test_knot(i16 %a0) { +; CHECK-LABEL: test_knot: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k0 +; CHECK-NEXT: knotw %k0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq %res = call i16 @llvm.x86.avx512.knot.w(i16 %a0) ret i16 %res } declare i16 @llvm.x86.avx512.kunpck.bw(i16, i16) nounwind readnone -; CHECK-LABEL: unpckbw_test -; CHECK: kunpckbw -; CHECK:ret define i16 @unpckbw_test(i16 %a0, i16 %a1) { +; CHECK-LABEL: unpckbw_test: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k0 +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: kunpckbw %k1, %k0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq %res = call i16 @llvm.x86.avx512.kunpck.bw(i16 %a0, i16 %a1) ret i16 %res } define <16 x float> @test_rcp_ps_512(<16 x float> %a0) { - ; CHECK: vrcp14ps {{.*}}encoding: [0x62,0xf2,0x7d,0x48,0x4c,0xc0] +; CHECK-LABEL: test_rcp_ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vrcp14ps %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.rcp14.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1) ; <<16 x float>> [#uses=1] ret <16 x float> %res } declare <16 x float> @llvm.x86.avx512.rcp14.ps.512(<16 x float>, <16 x float>, i16) nounwind readnone define <8 x double> @test_rcp_pd_512(<8 x double> %a0) { - ; CHECK: vrcp14pd {{.*}}encoding: [0x62,0xf2,0xfd,0x48,0x4c,0xc0] +; CHECK-LABEL: test_rcp_pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vrcp14pd %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x double> @llvm.x86.avx512.rcp14.pd.512(<8 x double> %a0, <8 x double> zeroinitializer, i8 -1) ; <<8 x double>> [#uses=1] ret <8 x double> %res } @@ -63,7 +97,10 @@ declare <8 x double> @llvm.x86.avx512.rcp14.pd.512(<8 x double>, <8 x double>, i declare <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512(<8 x double>, i32, <8 x double>, i8, i32) define <8 x double> @test7(<8 x double> %a) { -; CHECK: vrndscalepd {{.*}}encoding: [0x62,0xf3,0xfd,0x48,0x09,0xc0,0x0b] +; CHECK-LABEL: test7: +; CHECK: ## BB#0: +; CHECK-NEXT: vrndscalepd $11, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512(<8 x double> %a, i32 11, <8 x double> %a, i8 -1, i32 4) ret <8 x double>%res } @@ -71,121 +108,246 @@ define <8 x double> @test7(<8 x double> %a) { declare <16 x float> @llvm.x86.avx512.mask.rndscale.ps.512(<16 x float>, i32, <16 x float>, i16, i32) define <16 x float> @test8(<16 x float> %a) { -; CHECK: vrndscaleps {{.*}}encoding: [0x62,0xf3,0x7d,0x48,0x08,0xc0,0x0b] +; CHECK-LABEL: test8: +; CHECK: ## BB#0: +; CHECK-NEXT: vrndscaleps $11, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.rndscale.ps.512(<16 x float> %a, i32 11, <16 x float> %a, i16 -1, i32 4) ret <16 x float>%res } define <16 x float> @test_rsqrt_ps_512(<16 x float> %a0) { - ; CHECK: vrsqrt14ps {{.*}}encoding: [0x62,0xf2,0x7d,0x48,0x4e,0xc0] +; CHECK-LABEL: test_rsqrt_ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vrsqrt14ps %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.rsqrt14.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1) ; <<16 x float>> [#uses=1] ret <16 x float> %res } declare <16 x float> @llvm.x86.avx512.rsqrt14.ps.512(<16 x float>, <16 x float>, i16) nounwind readnone define <4 x float> @test_rsqrt14_ss(<4 x float> %a0) { - ; CHECK: vrsqrt14ss {{.*}}encoding: [0x62,0xf2,0x7d,0x08,0x4f,0xc0] +; CHECK-LABEL: test_rsqrt14_ss: +; CHECK: ## BB#0: +; CHECK-NEXT: vrsqrt14ss %xmm0, %xmm0, %xmm0 +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.rsqrt14.ss(<4 x float> %a0, <4 x float> %a0, <4 x float> zeroinitializer, i8 -1) ; <<4 x float>> [#uses=1] ret <4 x float> %res } declare <4 x float> @llvm.x86.avx512.rsqrt14.ss(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone define <4 x float> @test_rcp14_ss(<4 x float> %a0) { - ; CHECK: vrcp14ss {{.*}}encoding: [0x62,0xf2,0x7d,0x08,0x4d,0xc0] +; CHECK-LABEL: test_rcp14_ss: +; CHECK: ## BB#0: +; CHECK-NEXT: vrcp14ss %xmm0, %xmm0, %xmm0 +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.rcp14.ss(<4 x float> %a0, <4 x float> %a0, <4 x float> zeroinitializer, i8 -1) ; <<4 x float>> [#uses=1] ret <4 x float> %res } declare <4 x float> @llvm.x86.avx512.rcp14.ss(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone define <8 x double> @test_sqrt_pd_512(<8 x double> %a0) { - ; CHECK-LABEL: test_sqrt_pd_512 - ; CHECK: vsqrtpd - %res = call <8 x double> @llvm.x86.avx512.mask.sqrt.pd.512(<8 x double> %a0, <8 x double> zeroinitializer, i8 -1, i32 4) +; CHECK-LABEL: test_sqrt_pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vsqrtpd %zmm0, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.sqrt.pd.512(<8 x double> %a0, <8 x double> zeroinitializer, i8 -1, i32 4) ret <8 x double> %res } declare <8 x double> @llvm.x86.avx512.mask.sqrt.pd.512(<8 x double>, <8 x double>, i8, i32) nounwind readnone define <16 x float> @test_sqrt_ps_512(<16 x float> %a0) { - ; CHECK-LABEL: test_sqrt_ps_512 - ; CHECK: vsqrtps - %res = call <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 4) +; CHECK-LABEL: test_sqrt_ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vsqrtps %zmm0, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 4) ret <16 x float> %res } define <16 x float> @test_sqrt_round_ps_512(<16 x float> %a0) { - ; CHECK-LABEL: test_sqrt_round_ps_512 - ; CHECK: vsqrtps {rz-sae} - %res = call <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 3) +; CHECK-LABEL: test_sqrt_round_ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vsqrtps {rz-sae}, %zmm0, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 3) ret <16 x float> %res } declare <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float>, <16 x float>, i16, i32) nounwind readnone define <8 x double> @test_getexp_pd_512(<8 x double> %a0) { - ; CHECK-LABEL: test_getexp_pd_512 - ; CHECK: vgetexppd - %res = call <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double> %a0, <8 x double> zeroinitializer, i8 -1, i32 4) +; CHECK-LABEL: test_getexp_pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vgetexppd %zmm0, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double> %a0, <8 x double> zeroinitializer, i8 -1, i32 4) ret <8 x double> %res } define <8 x double> @test_getexp_round_pd_512(<8 x double> %a0) { - ; CHECK-LABEL: test_getexp_round_pd_512 - ; CHECK: vgetexppd {sae} - %res = call <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double> %a0, <8 x double> zeroinitializer, i8 -1, i32 8) +; CHECK-LABEL: test_getexp_round_pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vgetexppd {sae}, %zmm0, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double> %a0, <8 x double> zeroinitializer, i8 -1, i32 8) ret <8 x double> %res } declare <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double>, <8 x double>, i8, i32) nounwind readnone define <16 x float> @test_getexp_ps_512(<16 x float> %a0) { - ; CHECK-LABEL: test_getexp_ps_512 - ; CHECK: vgetexpps - %res = call <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 4) +; CHECK-LABEL: test_getexp_ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vgetexpps %zmm0, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 4) ret <16 x float> %res } define <16 x float> @test_getexp_round_ps_512(<16 x float> %a0) { - ; CHECK-LABEL: test_getexp_round_ps_512 - ; CHECK: vgetexpps {sae} - %res = call <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 8) +; CHECK-LABEL: test_getexp_round_ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vgetexpps {sae}, %zmm0, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 8) ret <16 x float> %res } declare <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float>, <16 x float>, i16, i32) nounwind readnone -define <4 x float> @test_sqrt_ss(<4 x float> %a0, <4 x float> %a1) { - ; CHECK: vsqrtss {{.*}}encoding: [0x62 - %res = call <4 x float> @llvm.x86.avx512.sqrt.ss(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1] +declare <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone + +define <4 x float> @test_sqrt_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) { +; CHECK-LABEL: test_sqrt_ss: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovaps %zmm2, %zmm3 +; CHECK-NEXT: vsqrtss %xmm1, %xmm0, %xmm3 {%k1} +; CHECK-NEXT: vsqrtss {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vsqrtss {ru-sae}, %xmm1, %xmm0, %xmm4 {%k1} {z} +; CHECK-NEXT: vsqrtss {rz-sae}, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vaddps %xmm2, %xmm3, %xmm1 +; CHECK-NEXT: vaddps %xmm0, %xmm4, %xmm0 +; CHECK-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res0 = call <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 4) + %res1 = call <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 1) + %res2 = call <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 2) + %res3 = call <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 3) + + %res.1 = fadd <4 x float> %res0, %res1 + %res.2 = fadd <4 x float> %res2, %res3 + %res = fadd <4 x float> %res.1, %res.2 ret <4 x float> %res } -declare <4 x float> @llvm.x86.avx512.sqrt.ss(<4 x float>, <4 x float>) nounwind readnone -define <2 x double> @test_sqrt_sd(<2 x double> %a0, <2 x double> %a1) { - ; CHECK: vsqrtsd {{.*}}encoding: [0x62 - %res = call <2 x double> @llvm.x86.avx512.sqrt.sd(<2 x double> %a0, <2 x double> %a1) ; <<2 x double>> [#uses=1] +declare <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone + +define <2 x double> @test_sqrt_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) { +; CHECK-LABEL: test_sqrt_sd: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovaps %zmm2, %zmm3 +; CHECK-NEXT: vsqrtsd %xmm1, %xmm0, %xmm3 {%k1} +; CHECK-NEXT: vsqrtsd {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vsqrtsd {ru-sae}, %xmm1, %xmm0, %xmm4 {%k1} {z} +; CHECK-NEXT: vsqrtsd {rz-sae}, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vaddpd %xmm2, %xmm3, %xmm1 +; CHECK-NEXT: vaddpd %xmm0, %xmm4, %xmm0 +; CHECK-NEXT: vaddpd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res0 = call <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 4) + %res1 = call <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 1) + %res2 = call <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 2) + %res3 = call <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 3) + + %res.1 = fadd <2 x double> %res0, %res1 + %res.2 = fadd <2 x double> %res2, %res3 + %res = fadd <2 x double> %res.1, %res.2 ret <2 x double> %res } -declare <2 x double> @llvm.x86.avx512.sqrt.sd(<2 x double>, <2 x double>) nounwind readnone define i64 @test_x86_sse2_cvtsd2si64(<2 x double> %a0) { - ; CHECK: vcvtsd2si {{.*}}encoding: [0x62 +; CHECK-LABEL: test_x86_sse2_cvtsd2si64: +; CHECK: ## BB#0: +; CHECK-NEXT: vcvtsd2si %xmm0, %rax +; CHECK-NEXT: retq %res = call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> %a0) ; [#uses=1] ret i64 %res } declare i64 @llvm.x86.sse2.cvtsd2si64(<2 x double>) nounwind readnone define <2 x double> @test_x86_sse2_cvtsi642sd(<2 x double> %a0, i64 %a1) { - ; CHECK: vcvtsi2sdq {{.*}}encoding: [0x62 +; CHECK-LABEL: test_x86_sse2_cvtsi642sd: +; CHECK: ## BB#0: +; CHECK-NEXT: vcvtsi2sdq %rdi, %xmm0, %xmm0 +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.sse2.cvtsi642sd(<2 x double> %a0, i64 %a1) ; <<2 x double>> [#uses=1] ret <2 x double> %res } declare <2 x double> @llvm.x86.sse2.cvtsi642sd(<2 x double>, i64) nounwind readnone -define i64 @test_x86_sse2_cvttsd2si64(<2 x double> %a0) { - ; CHECK: vcvttsd2si {{.*}}encoding: [0x62 - %res = call i64 @llvm.x86.sse2.cvttsd2si64(<2 x double> %a0) ; [#uses=1] - ret i64 %res +define i64 @test_x86_avx512_cvttsd2si64(<2 x double> %a0) { +; CHECK-LABEL: test_x86_avx512_cvttsd2si64: +; CHECK: ## BB#0: +; CHECK-NEXT: vcvttsd2si %xmm0, %rcx +; CHECK-NEXT: vcvttsd2si {sae}, %xmm0, %rax +; CHECK-NEXT: addq %rcx, %rax +; CHECK-NEXT: retq + %res0 = call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> %a0, i32 4) ; + %res1 = call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> %a0, i32 8) ; + %res2 = add i64 %res0, %res1 + ret i64 %res2 +} +declare i64 @llvm.x86.avx512.cvttsd2si64(<2 x double>, i32) nounwind readnone + +define i32 @test_x86_avx512_cvttsd2usi(<2 x double> %a0) { +; CHECK-LABEL: test_x86_avx512_cvttsd2usi: +; CHECK: ## BB#0: +; CHECK-NEXT: vcvttsd2usi %xmm0, %ecx +; CHECK-NEXT: vcvttsd2usi {sae}, %xmm0, %eax +; CHECK-NEXT: addl %ecx, %eax +; CHECK-NEXT: retq + %res0 = call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> %a0, i32 4) ; + %res1 = call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> %a0, i32 8) ; + %res2 = add i32 %res0, %res1 + ret i32 %res2 +} +declare i32 @llvm.x86.avx512.cvttsd2usi(<2 x double>, i32) nounwind readnone + +define i32 @test_x86_avx512_cvttsd2si(<2 x double> %a0) { +; CHECK-LABEL: test_x86_avx512_cvttsd2si: +; CHECK: ## BB#0: +; CHECK-NEXT: vcvttsd2si %xmm0, %ecx +; CHECK-NEXT: vcvttsd2si {sae}, %xmm0, %eax +; CHECK-NEXT: addl %ecx, %eax +; CHECK-NEXT: retq + %res0 = call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> %a0, i32 4) ; + %res1 = call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> %a0, i32 8) ; + %res2 = add i32 %res0, %res1 + ret i32 %res2 } -declare i64 @llvm.x86.sse2.cvttsd2si64(<2 x double>) nounwind readnone +declare i32 @llvm.x86.avx512.cvttsd2si(<2 x double>, i32) nounwind readnone + +define i64 @test_x86_avx512_cvttsd2usi64(<2 x double> %a0) { +; CHECK-LABEL: test_x86_avx512_cvttsd2usi64: +; CHECK: ## BB#0: +; CHECK-NEXT: vcvttsd2usi %xmm0, %rcx +; CHECK-NEXT: vcvttsd2usi {sae}, %xmm0, %rax +; CHECK-NEXT: addq %rcx, %rax +; CHECK-NEXT: retq + %res0 = call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> %a0, i32 4) ; + %res1 = call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> %a0, i32 8) ; + %res2 = add i64 %res0, %res1 + ret i64 %res2 +} +declare i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double>, i32) nounwind readnone + define i64 @test_x86_sse_cvtss2si64(<4 x float> %a0) { - ; CHECK: vcvtss2si {{.*}}encoding: [0x62 +; CHECK-LABEL: test_x86_sse_cvtss2si64: +; CHECK: ## BB#0: +; CHECK-NEXT: vcvtss2si %xmm0, %rax +; CHECK-NEXT: retq %res = call i64 @llvm.x86.sse.cvtss2si64(<4 x float> %a0) ; [#uses=1] ret i64 %res } @@ -193,37 +355,139 @@ declare i64 @llvm.x86.sse.cvtss2si64(<4 x float>) nounwind readnone define <4 x float> @test_x86_sse_cvtsi642ss(<4 x float> %a0, i64 %a1) { - ; CHECK: vcvtsi2ssq {{.*}}encoding: [0x62 +; CHECK-LABEL: test_x86_sse_cvtsi642ss: +; CHECK: ## BB#0: +; CHECK-NEXT: vcvtsi2ssq %rdi, %xmm0, %xmm0 +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.sse.cvtsi642ss(<4 x float> %a0, i64 %a1) ; <<4 x float>> [#uses=1] ret <4 x float> %res } declare <4 x float> @llvm.x86.sse.cvtsi642ss(<4 x float>, i64) nounwind readnone -define i64 @test_x86_sse_cvttss2si64(<4 x float> %a0) { - ; CHECK: vcvttss2si {{.*}}encoding: [0x62 - %res = call i64 @llvm.x86.sse.cvttss2si64(<4 x float> %a0) ; [#uses=1] - ret i64 %res +define i32 @test_x86_avx512_cvttss2si(<4 x float> %a0) { +; CHECK-LABEL: test_x86_avx512_cvttss2si: +; CHECK: ## BB#0: +; CHECK-NEXT: vcvttss2si {sae}, %xmm0, %ecx +; CHECK-NEXT: vcvttss2si %xmm0, %eax +; CHECK-NEXT: addl %ecx, %eax +; CHECK-NEXT: retq + %res0 = call i32 @llvm.x86.avx512.cvttss2si(<4 x float> %a0, i32 8) ; + %res1 = call i32 @llvm.x86.avx512.cvttss2si(<4 x float> %a0, i32 4) ; + %res2 = add i32 %res0, %res1 + ret i32 %res2 +} +declare i32 @llvm.x86.avx512.cvttss2si(<4 x float>, i32) nounwind readnone + +define i64 @test_x86_avx512_cvttss2si64(<4 x float> %a0) { +; CHECK-LABEL: test_x86_avx512_cvttss2si64: +; CHECK: ## BB#0: +; CHECK-NEXT: vcvttss2si %xmm0, %rcx +; CHECK-NEXT: vcvttss2si {sae}, %xmm0, %rax +; CHECK-NEXT: addq %rcx, %rax +; CHECK-NEXT: retq + %res0 = call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> %a0, i32 4) ; + %res1 = call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> %a0, i32 8) ; + %res2 = add i64 %res0, %res1 + ret i64 %res2 +} +declare i64 @llvm.x86.avx512.cvttss2si64(<4 x float>, i32) nounwind readnone + +define i32 @test_x86_avx512_cvttss2usi(<4 x float> %a0) { +; CHECK-LABEL: test_x86_avx512_cvttss2usi: +; CHECK: ## BB#0: +; CHECK-NEXT: vcvttss2usi {sae}, %xmm0, %ecx +; CHECK-NEXT: vcvttss2usi %xmm0, %eax +; CHECK-NEXT: addl %ecx, %eax +; CHECK-NEXT: retq + %res0 = call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> %a0, i32 8) ; + %res1 = call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> %a0, i32 4) ; + %res2 = add i32 %res0, %res1 + ret i32 %res2 +} +declare i32 @llvm.x86.avx512.cvttss2usi(<4 x float>, i32) nounwind readnone + +define i64 @test_x86_avx512_cvttss2usi64(<4 x float> %a0) { +; CHECK-LABEL: test_x86_avx512_cvttss2usi64: +; CHECK: ## BB#0: +; CHECK-NEXT: vcvttss2usi %xmm0, %rcx +; CHECK-NEXT: vcvttss2usi {sae}, %xmm0, %rax +; CHECK-NEXT: addq %rcx, %rax +; CHECK-NEXT: retq + %res0 = call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> %a0, i32 4) ; + %res1 = call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> %a0, i32 8) ; + %res2 = add i64 %res0, %res1 + ret i64 %res2 } -declare i64 @llvm.x86.sse.cvttss2si64(<4 x float>) nounwind readnone +declare i64 @llvm.x86.avx512.cvttss2usi64(<4 x float>, i32) nounwind readnone define i64 @test_x86_avx512_cvtsd2usi64(<2 x double> %a0) { - ; CHECK: vcvtsd2usi {{.*}}encoding: [0x62 +; CHECK-LABEL: test_x86_avx512_cvtsd2usi64: +; CHECK: ## BB#0: +; CHECK-NEXT: vcvtsd2usi %xmm0, %rax +; CHECK-NEXT: retq %res = call i64 @llvm.x86.avx512.cvtsd2usi64(<2 x double> %a0) ; [#uses=1] ret i64 %res } declare i64 @llvm.x86.avx512.cvtsd2usi64(<2 x double>) nounwind readnone define <16 x float> @test_x86_vcvtph2ps_512(<16 x i16> %a0) { - ; CHECK: vcvtph2ps %ymm0, %zmm0 ## encoding: [0x62,0xf2,0x7d,0x48,0x13,0xc0] +; CHECK-LABEL: test_x86_vcvtph2ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vcvtph2ps %ymm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> %a0, <16 x float> zeroinitializer, i16 -1, i32 4) ret <16 x float> %res } + +define <16 x float> @test_x86_vcvtph2ps_512_sae(<16 x i16> %a0) { +; CHECK-LABEL: test_x86_vcvtph2ps_512_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vcvtph2ps {sae}, %ymm0, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> %a0, <16 x float> zeroinitializer, i16 -1, i32 8) + ret <16 x float> %res +} + +define <16 x float> @test_x86_vcvtph2ps_512_rrk(<16 x i16> %a0,<16 x float> %a1, i16 %mask) { +; CHECK-LABEL: test_x86_vcvtph2ps_512_rrk: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vcvtph2ps %ymm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> %a0, <16 x float> %a1, i16 %mask, i32 4) + ret <16 x float> %res +} + +define <16 x float> @test_x86_vcvtph2ps_512_sae_rrkz(<16 x i16> %a0, i16 %mask) { +; CHECK-LABEL: test_x86_vcvtph2ps_512_sae_rrkz: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vcvtph2ps {sae}, %ymm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> %a0, <16 x float> zeroinitializer, i16 %mask, i32 8) + ret <16 x float> %res +} + +define <16 x float> @test_x86_vcvtph2ps_512_rrkz(<16 x i16> %a0, i16 %mask) { +; CHECK-LABEL: test_x86_vcvtph2ps_512_rrkz: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vcvtph2ps %ymm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> %a0, <16 x float> zeroinitializer, i16 %mask, i32 4) + ret <16 x float> %res +} + declare <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16>, <16 x float>, i16, i32) nounwind readonly define <16 x i16> @test_x86_vcvtps2ph_256(<16 x float> %a0) { - ; CHECK: vcvtps2ph $2, %zmm0, %ymm0 ## encoding: [0x62,0xf3,0x7d,0x48,0x1d,0xc0,0x02] +; CHECK-LABEL: test_x86_vcvtps2ph_256: +; CHECK: ## BB#0: +; CHECK-NEXT: vcvtps2ph $2, %zmm0, %ymm0 +; CHECK-NEXT: retq %res = call <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float> %a0, i32 2, <16 x i16> zeroinitializer, i16 -1) ret <16 x i16> %res } @@ -231,65 +495,124 @@ define <16 x i16> @test_x86_vcvtps2ph_256(<16 x float> %a0) { declare <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float>, i32, <16 x i16>, i16) nounwind readonly define <16 x float> @test_x86_vbroadcast_ss_512(i8* %a0) { - ; CHECK: vbroadcastss +; CHECK-LABEL: test_x86_vbroadcast_ss_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastss (%rdi), %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.vbroadcast.ss.512(i8* %a0) ; <<16 x float>> [#uses=1] ret <16 x float> %res } declare <16 x float> @llvm.x86.avx512.vbroadcast.ss.512(i8*) nounwind readonly define <8 x double> @test_x86_vbroadcast_sd_512(i8* %a0) { - ; CHECK: vbroadcastsd +; CHECK-LABEL: test_x86_vbroadcast_sd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vbroadcastsd (%rdi), %zmm0 +; CHECK-NEXT: retq %res = call <8 x double> @llvm.x86.avx512.vbroadcast.sd.512(i8* %a0) ; <<8 x double>> [#uses=1] ret <8 x double> %res } declare <8 x double> @llvm.x86.avx512.vbroadcast.sd.512(i8*) nounwind readonly -define <16 x float> @test_x86_vbroadcast_ss_ps_512(<4 x float> %a0) { - ; CHECK: vbroadcastss - %res = call <16 x float> @llvm.x86.avx512.vbroadcast.ss.ps.512(<4 x float> %a0) ; <<16 x float>> [#uses=1] - ret <16 x float> %res -} -declare <16 x float> @llvm.x86.avx512.vbroadcast.ss.ps.512(<4 x float>) nounwind readonly - -define <8 x double> @test_x86_vbroadcast_sd_pd_512(<2 x double> %a0) { - ; CHECK: vbroadcastsd - %res = call <8 x double> @llvm.x86.avx512.vbroadcast.sd.pd.512(<2 x double> %a0) ; <<8 x double>> [#uses=1] - ret <8 x double> %res -} -declare <8 x double> @llvm.x86.avx512.vbroadcast.sd.pd.512(<2 x double>) nounwind readonly - -define <16 x i32> @test_x86_pbroadcastd_512(<4 x i32> %a0) { - ; CHECK: vpbroadcastd - %res = call <16 x i32> @llvm.x86.avx512.pbroadcastd.512(<4 x i32> %a0) ; <<16 x i32>> [#uses=1] - ret <16 x i32> %res +define <16 x float> @test_x86_vbroadcast_ss_ps_512(<4 x float> %a0, <16 x float> %a1, i16 %mask ) { +; CHECK-LABEL: test_x86_vbroadcast_ss_ps_512: +; CHECK: kmovw %edi, %k1 +; CHECK-NEXT: vbroadcastss %xmm0, %zmm1 {%k1} +; CHECK-NEXT: vbroadcastss %xmm0, %zmm2 {%k1} {z} +; CHECK-NEXT: vbroadcastss %xmm0, %zmm0 +; CHECK-NEXT: vaddps %zmm1, %zmm0, %zmm0 + + %res = call <16 x float> @llvm.x86.avx512.mask.broadcast.ss.ps.512(<4 x float> %a0, <16 x float> zeroinitializer, i16 -1) + %res1 = call <16 x float> @llvm.x86.avx512.mask.broadcast.ss.ps.512(<4 x float> %a0, <16 x float> %a1, i16 %mask) + %res2 = call <16 x float> @llvm.x86.avx512.mask.broadcast.ss.ps.512(<4 x float> %a0, <16 x float> zeroinitializer, i16 %mask) + %res3 = fadd <16 x float> %res, %res1 + %res4 = fadd <16 x float> %res2, %res3 + ret <16 x float> %res4 +} +declare <16 x float> @llvm.x86.avx512.mask.broadcast.ss.ps.512(<4 x float>, <16 x float>, i16) nounwind readonly + + +define <8 x double> @test_x86_vbroadcast_sd_pd_512(<2 x double> %a0, <8 x double> %a1, i8 %mask ) { +; CHECK-LABEL: test_x86_vbroadcast_sd_pd_512: +; CHECK: kmovw %eax, %k1 +; CHECK-NEXT: vbroadcastsd %xmm0, %zmm1 {%k1} +; CHECK-NEXT: vbroadcastsd %xmm0, %zmm2 {%k1} {z} +; CHECK-NEXT: vbroadcastsd %xmm0, %zmm0 +; CHECK-NEXT: vaddpd %zmm1, %zmm0, %zmm0 + + %res = call <8 x double> @llvm.x86.avx512.mask.broadcast.sd.pd.512(<2 x double> %a0, <8 x double> zeroinitializer, i8 -1) + %res1 = call <8 x double> @llvm.x86.avx512.mask.broadcast.sd.pd.512(<2 x double> %a0, <8 x double> %a1, i8 %mask) + %res2 = call <8 x double> @llvm.x86.avx512.mask.broadcast.sd.pd.512(<2 x double> %a0, <8 x double> zeroinitializer, i8 %mask) + %res3 = fadd <8 x double> %res, %res1 + %res4 = fadd <8 x double> %res2, %res3 + ret <8 x double> %res4 +} +declare <8 x double> @llvm.x86.avx512.mask.broadcast.sd.pd.512(<2 x double>, <8 x double>, i8) nounwind readonly + +define <16 x i32>@test_int_x86_avx512_pbroadcastd_512(<4 x i32> %x0, <16 x i32> %x1, i16 %mask) { +; CHECK-LABEL: test_int_x86_avx512_pbroadcastd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpbroadcastd %xmm0, %zmm1 {%k1} +; CHECK-NEXT: vpbroadcastd %xmm0, %zmm2 {%k1} {z} +; CHECK-NEXT: vpbroadcastd %xmm0, %zmm0 +; CHECK-NEXT: vpaddd %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: vpaddd %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x i32> @llvm.x86.avx512.pbroadcastd.512(<4 x i32> %x0, <16 x i32> %x1, i16 -1) + %res1 = call <16 x i32> @llvm.x86.avx512.pbroadcastd.512(<4 x i32> %x0, <16 x i32> %x1, i16 %mask) + %res2 = call <16 x i32> @llvm.x86.avx512.pbroadcastd.512(<4 x i32> %x0, <16 x i32> zeroinitializer, i16 %mask) + %res3 = add <16 x i32> %res, %res1 + %res4 = add <16 x i32> %res2, %res3 + ret <16 x i32> %res4 } -declare <16 x i32> @llvm.x86.avx512.pbroadcastd.512(<4 x i32>) nounwind readonly +declare <16 x i32> @llvm.x86.avx512.pbroadcastd.512(<4 x i32>, <16 x i32>, i16) define <16 x i32> @test_x86_pbroadcastd_i32_512(i32 %a0) { - ; CHECK: vpbroadcastd +; CHECK-LABEL: test_x86_pbroadcastd_i32_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpbroadcastd %edi, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.pbroadcastd.i32.512(i32 %a0) ; <<16 x i32>> [#uses=1] ret <16 x i32> %res } declare <16 x i32> @llvm.x86.avx512.pbroadcastd.i32.512(i32) nounwind readonly -define <8 x i64> @test_x86_pbroadcastq_512(<2 x i64> %a0) { - ; CHECK: vpbroadcastq - %res = call <8 x i64> @llvm.x86.avx512.pbroadcastq.512(<2 x i64> %a0) ; <<8 x i64>> [#uses=1] - ret <8 x i64> %res +define <8 x i64>@test_int_x86_avx512_pbroadcastq_512(<2 x i64> %x0, <8 x i64> %x1, i8 %mask) { +; CHECK-LABEL: test_int_x86_avx512_pbroadcastq_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpbroadcastq %xmm0, %zmm1 {%k1} +; CHECK-NEXT: vpbroadcastq %xmm0, %zmm2 {%k1} {z} +; CHECK-NEXT: vpbroadcastq %xmm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.pbroadcastq.512(<2 x i64> %x0, <8 x i64> %x1,i8 -1) + %res1 = call <8 x i64> @llvm.x86.avx512.pbroadcastq.512(<2 x i64> %x0, <8 x i64> %x1,i8 %mask) + %res2 = call <8 x i64> @llvm.x86.avx512.pbroadcastq.512(<2 x i64> %x0, <8 x i64> zeroinitializer,i8 %mask) + %res3 = add <8 x i64> %res, %res1 + %res4 = add <8 x i64> %res2, %res3 + ret <8 x i64> %res4 } -declare <8 x i64> @llvm.x86.avx512.pbroadcastq.512(<2 x i64>) nounwind readonly +declare <8 x i64> @llvm.x86.avx512.pbroadcastq.512(<2 x i64>, <8 x i64>, i8) define <8 x i64> @test_x86_pbroadcastq_i64_512(i64 %a0) { - ; CHECK: vpbroadcastq +; CHECK-LABEL: test_x86_pbroadcastq_i64_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpbroadcastq %rdi, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.pbroadcastq.i64.512(i64 %a0) ; <<8 x i64>> [#uses=1] ret <8 x i64> %res } declare <8 x i64> @llvm.x86.avx512.pbroadcastq.i64.512(i64) nounwind readonly define <16 x i32> @test_conflict_d(<16 x i32> %a) { - ; CHECK: movw $-1, %ax - ; CHECK: vpxor - ; CHECK: vpconflictd +; CHECK-LABEL: test_conflict_d: +; CHECK: ## BB#0: +; CHECK-NEXT: vpconflictd %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.conflict.d.512(<16 x i32> %a, <16 x i32> zeroinitializer, i16 -1) ret <16 x i32> %res } @@ -297,9 +620,10 @@ define <16 x i32> @test_conflict_d(<16 x i32> %a) { declare <16 x i32> @llvm.x86.avx512.mask.conflict.d.512(<16 x i32>, <16 x i32>, i16) nounwind readonly define <8 x i64> @test_conflict_q(<8 x i64> %a) { - ; CHECK: movb $-1, %al - ; CHECK: vpxor - ; CHECK: vpconflictq +; CHECK-LABEL: test_conflict_q: +; CHECK: ## BB#0: +; CHECK-NEXT: vpconflictq %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.conflict.q.512(<8 x i64> %a, <8 x i64> zeroinitializer, i8 -1) ret <8 x i64> %res } @@ -307,21 +631,32 @@ define <8 x i64> @test_conflict_q(<8 x i64> %a) { declare <8 x i64> @llvm.x86.avx512.mask.conflict.q.512(<8 x i64>, <8 x i64>, i8) nounwind readonly define <16 x i32> @test_maskz_conflict_d(<16 x i32> %a, i16 %mask) { - ; CHECK: vpconflictd +; CHECK-LABEL: test_maskz_conflict_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpconflictd %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.conflict.d.512(<16 x i32> %a, <16 x i32> zeroinitializer, i16 %mask) ret <16 x i32> %res } define <8 x i64> @test_mask_conflict_q(<8 x i64> %a, <8 x i64> %b, i8 %mask) { - ; CHECK: vpconflictq +; CHECK-LABEL: test_mask_conflict_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpconflictq %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.conflict.q.512(<8 x i64> %a, <8 x i64> %b, i8 %mask) ret <8 x i64> %res } define <16 x i32> @test_lzcnt_d(<16 x i32> %a) { - ; CHECK: movw $-1, %ax - ; CHECK: vpxor - ; CHECK: vplzcntd +; CHECK-LABEL: test_lzcnt_d: +; CHECK: ## BB#0: +; CHECK-NEXT: vplzcntd %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.lzcnt.d.512(<16 x i32> %a, <16 x i32> zeroinitializer, i16 -1) ret <16 x i32> %res } @@ -329,9 +664,10 @@ define <16 x i32> @test_lzcnt_d(<16 x i32> %a) { declare <16 x i32> @llvm.x86.avx512.mask.lzcnt.d.512(<16 x i32>, <16 x i32>, i16) nounwind readonly define <8 x i64> @test_lzcnt_q(<8 x i64> %a) { - ; CHECK: movb $-1, %al - ; CHECK: vpxor - ; CHECK: vplzcntq +; CHECK-LABEL: test_lzcnt_q: +; CHECK: ## BB#0: +; CHECK-NEXT: vplzcntq %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.lzcnt.q.512(<8 x i64> %a, <8 x i64> zeroinitializer, i8 -1) ret <8 x i64> %res } @@ -340,37 +676,34 @@ declare <8 x i64> @llvm.x86.avx512.mask.lzcnt.q.512(<8 x i64>, <8 x i64>, i8) no define <16 x i32> @test_mask_lzcnt_d(<16 x i32> %a, <16 x i32> %b, i16 %mask) { - ; CHECK: vplzcntd +; CHECK-LABEL: test_mask_lzcnt_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vplzcntd %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.lzcnt.d.512(<16 x i32> %a, <16 x i32> %b, i16 %mask) ret <16 x i32> %res } define <8 x i64> @test_mask_lzcnt_q(<8 x i64> %a, <8 x i64> %b, i8 %mask) { - ; CHECK: vplzcntq +; CHECK-LABEL: test_mask_lzcnt_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vplzcntq %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.lzcnt.q.512(<8 x i64> %a, <8 x i64> %b, i8 %mask) ret <8 x i64> %res } -define <16 x i32> @test_ctlz_d(<16 x i32> %a) { - ; CHECK-LABEL: test_ctlz_d - ; CHECK: vplzcntd - %res = call <16 x i32> @llvm.ctlz.v16i32(<16 x i32> %a, i1 false) - ret <16 x i32> %res -} - -declare <16 x i32> @llvm.ctlz.v16i32(<16 x i32>, i1) nounwind readonly - -define <8 x i64> @test_ctlz_q(<8 x i64> %a) { - ; CHECK-LABEL: test_ctlz_q - ; CHECK: vplzcntq - %res = call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> %a, i1 false) - ret <8 x i64> %res -} - -declare <8 x i64> @llvm.ctlz.v8i64(<8 x i64>, i1) nounwind readonly - define <16 x float> @test_x86_mask_blend_ps_512(i16 %a0, <16 x float> %a1, <16 x float> %a2) { - ; CHECK: vblendmps %zmm1, %zmm0 +; CHECK-LABEL: test_x86_mask_blend_ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vblendmps %zmm1, %zmm0, %zmm0 {%k1} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.blend.ps.512(<16 x float> %a1, <16 x float> %a2, i16 %a0) ; <<16 x float>> [#uses=1] ret <16 x float> %res } @@ -378,14 +711,23 @@ define <16 x float> @test_x86_mask_blend_ps_512(i16 %a0, <16 x float> %a1, <16 x declare <16 x float> @llvm.x86.avx512.mask.blend.ps.512(<16 x float>, <16 x float>, i16) nounwind readonly define <8 x double> @test_x86_mask_blend_pd_512(i8 %a0, <8 x double> %a1, <8 x double> %a2) { - ; CHECK: vblendmpd %zmm1, %zmm0 +; CHECK-LABEL: test_x86_mask_blend_pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vblendmpd %zmm1, %zmm0, %zmm0 {%k1} +; CHECK-NEXT: retq %res = call <8 x double> @llvm.x86.avx512.mask.blend.pd.512(<8 x double> %a1, <8 x double> %a2, i8 %a0) ; <<8 x double>> [#uses=1] ret <8 x double> %res } define <8 x double> @test_x86_mask_blend_pd_512_memop(<8 x double> %a, <8 x double>* %ptr, i8 %mask) { - ; CHECK-LABEL: test_x86_mask_blend_pd_512_memop - ; CHECK: vblendmpd (% +; CHECK-LABEL: test_x86_mask_blend_pd_512_memop: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vblendmpd (%rdi), %zmm0, %zmm0 {%k1} +; CHECK-NEXT: retq %b = load <8 x double>, <8 x double>* %ptr %res = call <8 x double> @llvm.x86.avx512.mask.blend.pd.512(<8 x double> %a, <8 x double> %b, i8 %mask) ; <<8 x double>> [#uses=1] ret <8 x double> %res @@ -393,28 +735,45 @@ define <8 x double> @test_x86_mask_blend_pd_512_memop(<8 x double> %a, <8 x doub declare <8 x double> @llvm.x86.avx512.mask.blend.pd.512(<8 x double>, <8 x double>, i8) nounwind readonly define <16 x i32> @test_x86_mask_blend_d_512(i16 %a0, <16 x i32> %a1, <16 x i32> %a2) { - ; CHECK: vpblendmd +; CHECK-LABEL: test_x86_mask_blend_d_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpblendmd %zmm1, %zmm0, %zmm0 {%k1} +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.blend.d.512(<16 x i32> %a1, <16 x i32> %a2, i16 %a0) ; <<16 x i32>> [#uses=1] ret <16 x i32> %res } declare <16 x i32> @llvm.x86.avx512.mask.blend.d.512(<16 x i32>, <16 x i32>, i16) nounwind readonly define <8 x i64> @test_x86_mask_blend_q_512(i8 %a0, <8 x i64> %a1, <8 x i64> %a2) { - ; CHECK: vpblendmq +; CHECK-LABEL: test_x86_mask_blend_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpblendmq %zmm1, %zmm0, %zmm0 {%k1} +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.blend.q.512(<8 x i64> %a1, <8 x i64> %a2, i8 %a0) ; <<8 x i64>> [#uses=1] ret <8 x i64> %res } declare <8 x i64> @llvm.x86.avx512.mask.blend.q.512(<8 x i64>, <8 x i64>, i8) nounwind readonly define i16 @test_cmpps(<16 x float> %a, <16 x float> %b) { - ;CHECK: vcmpleps {sae}{{.*}}encoding: [0x62,0xf1,0x7c,0x18,0xc2,0xc1,0x02] +; CHECK-LABEL: test_cmpps: +; CHECK: ## BB#0: +; CHECK-NEXT: vcmpleps {sae}, %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq %res = call i16 @llvm.x86.avx512.mask.cmp.ps.512(<16 x float> %a, <16 x float> %b, i32 2, i16 -1, i32 8) ret i16 %res } declare i16 @llvm.x86.avx512.mask.cmp.ps.512(<16 x float> , <16 x float> , i32, i16, i32) define i8 @test_cmppd(<8 x double> %a, <8 x double> %b) { - ;CHECK: vcmpneqpd %zmm{{.*}}encoding: [0x62,0xf1,0xfd,0x48,0xc2,0xc1,0x04] +; CHECK-LABEL: test_cmppd: +; CHECK: ## BB#0: +; CHECK-NEXT: vcmpneqpd %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq %res = call i8 @llvm.x86.avx512.mask.cmp.pd.512(<8 x double> %a, <8 x double> %b, i32 4, i8 -1, i32 4) ret i8 %res } @@ -422,7 +781,10 @@ declare <8 x i64> @llvm.x86.avx512.mask.blend.q.512(<8 x i64>, <8 x i64>, i8) no ; fp min - max define <8 x double> @test_vmaxpd(<8 x double> %a0, <8 x double> %a1) { - ; CHECK: vmaxpd +; CHECK-LABEL: test_vmaxpd: +; CHECK: ## BB#0: +; CHECK-NEXT: vmaxpd %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x double> @llvm.x86.avx512.mask.max.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double>zeroinitializer, i8 -1, i32 4) ret <8 x double> %res @@ -431,7 +793,10 @@ declare <8 x double> @llvm.x86.avx512.mask.max.pd.512(<8 x double>, <8 x double> <8 x double>, i8, i32) define <8 x double> @test_vminpd(<8 x double> %a0, <8 x double> %a1) { - ; CHECK: vminpd +; CHECK-LABEL: test_vminpd: +; CHECK: ## BB#0: +; CHECK-NEXT: vminpd %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x double> @llvm.x86.avx512.mask.min.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double>zeroinitializer, i8 -1, i32 4) ret <8 x double> %res @@ -441,11 +806,14 @@ declare <8 x double> @llvm.x86.avx512.mask.min.pd.512(<8 x double>, <8 x double> declare <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32>, <16 x i32>, i16) -; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_d_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpabsd{{.*}}{%k1} define <16 x i32>@test_int_x86_avx512_mask_pabs_d_512(<16 x i32> %x0, <16 x i32> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pabs_d_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpabsd %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vpabsd %zmm0, %zmm0 +; CHECK-NEXT: vpaddd %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32> %x0, <16 x i32> %x1, i16 %x2) %res1 = call <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32> %x0, <16 x i32> %x1, i16 -1) %res2 = add <16 x i32> %res, %res1 @@ -454,11 +822,15 @@ define <16 x i32>@test_int_x86_avx512_mask_pabs_d_512(<16 x i32> %x0, <16 x i32> declare <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64>, <8 x i64>, i8) -; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_q_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpabsq{{.*}}{%k1} define <8 x i64>@test_int_x86_avx512_mask_pabs_q_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pabs_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpabsq %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vpabsq %zmm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2) %res1 = call <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64> %x0, <8 x i64> %x1, i8 -1) %res2 = add <8 x i64> %res, %res1 @@ -466,21 +838,33 @@ define <8 x i64>@test_int_x86_avx512_mask_pabs_q_512(<8 x i64> %x0, <8 x i64> %x } define i8 @test_vptestmq(<8 x i64> %a0, <8 x i64> %a1) { - ; CHECK: vptestmq {{.*}}encoding: [0x62,0xf2,0xfd,0x48,0x27,0xc1] +; CHECK-LABEL: test_vptestmq: +; CHECK: ## BB#0: +; CHECK-NEXT: vptestmq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq %res = call i8 @llvm.x86.avx512.mask.ptestm.q.512(<8 x i64> %a0, <8 x i64> %a1, i8 -1) ret i8 %res } declare i8 @llvm.x86.avx512.mask.ptestm.q.512(<8 x i64>, <8 x i64>, i8) define i16 @test_vptestmd(<16 x i32> %a0, <16 x i32> %a1) { - ; CHECK: vptestmd {{.*}}encoding: [0x62,0xf2,0x7d,0x48,0x27,0xc1] +; CHECK-LABEL: test_vptestmd: +; CHECK: ## BB#0: +; CHECK-NEXT: vptestmd %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq %res = call i16 @llvm.x86.avx512.mask.ptestm.d.512(<16 x i32> %a0, <16 x i32> %a1, i16 -1) ret i16 %res } declare i16 @llvm.x86.avx512.mask.ptestm.d.512(<16 x i32>, <16 x i32>, i16) define void @test_store1(<16 x float> %data, i8* %ptr, i16 %mask) { -; CHECK: vmovups {{.*}}encoding: [0x62,0xf1,0x7c,0x49,0x11,0x07] +; CHECK-LABEL: test_store1: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vmovups %zmm0, (%rdi) {%k1} +; CHECK-NEXT: retq call void @llvm.x86.avx512.mask.storeu.ps.512(i8* %ptr, <16 x float> %data, i16 %mask) ret void } @@ -488,7 +872,11 @@ define void @test_store1(<16 x float> %data, i8* %ptr, i16 %mask) { declare void @llvm.x86.avx512.mask.storeu.ps.512(i8*, <16 x float>, i16 ) define void @test_store2(<8 x double> %data, i8* %ptr, i8 %mask) { -; CHECK: vmovupd {{.*}}encoding: [0x62,0xf1,0xfd,0x49,0x11,0x07] +; CHECK-LABEL: test_store2: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vmovupd %zmm0, (%rdi) {%k1} +; CHECK-NEXT: retq call void @llvm.x86.avx512.mask.storeu.pd.512(i8* %ptr, <8 x double> %data, i8 %mask) ret void } @@ -565,32 +953,45 @@ declare <8 x i64> @llvm.x86.avx512.movntdqa(i8*) define <8 x i64> @test_valign_q(<8 x i64> %a, <8 x i64> %b) { ; CHECK-LABEL: test_valign_q: -; CHECK: valignq $2, %zmm1, %zmm0, %zmm0 - %res = call <8 x i64> @llvm.x86.avx512.mask.valign.q.512(<8 x i64> %a, <8 x i64> %b, i8 2, <8 x i64> zeroinitializer, i8 -1) +; CHECK: ## BB#0: +; CHECK-NEXT: valignq $2, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.valign.q.512(<8 x i64> %a, <8 x i64> %b, i32 2, <8 x i64> zeroinitializer, i8 -1) ret <8 x i64> %res } define <8 x i64> @test_mask_valign_q(<8 x i64> %a, <8 x i64> %b, <8 x i64> %src, i8 %mask) { ; CHECK-LABEL: test_mask_valign_q: -; CHECK: valignq $2, %zmm1, %zmm0, %zmm2 {%k1} - %res = call <8 x i64> @llvm.x86.avx512.mask.valign.q.512(<8 x i64> %a, <8 x i64> %b, i8 2, <8 x i64> %src, i8 %mask) +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: valignq $2, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.valign.q.512(<8 x i64> %a, <8 x i64> %b, i32 2, <8 x i64> %src, i8 %mask) ret <8 x i64> %res } -declare <8 x i64> @llvm.x86.avx512.mask.valign.q.512(<8 x i64>, <8 x i64>, i8, <8 x i64>, i8) +declare <8 x i64> @llvm.x86.avx512.mask.valign.q.512(<8 x i64>, <8 x i64>, i32, <8 x i64>, i8) define <16 x i32> @test_maskz_valign_d(<16 x i32> %a, <16 x i32> %b, i16 %mask) { ; CHECK-LABEL: test_maskz_valign_d: -; CHECK: valignd $5, %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf3,0x7d,0xc9,0x03,0xc1,0x05] - %res = call <16 x i32> @llvm.x86.avx512.mask.valign.d.512(<16 x i32> %a, <16 x i32> %b, i8 5, <16 x i32> zeroinitializer, i16 %mask) +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: valignd $5, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq + %res = call <16 x i32> @llvm.x86.avx512.mask.valign.d.512(<16 x i32> %a, <16 x i32> %b, i32 5, <16 x i32> zeroinitializer, i16 %mask) ret <16 x i32> %res } -declare <16 x i32> @llvm.x86.avx512.mask.valign.d.512(<16 x i32>, <16 x i32>, i8, <16 x i32>, i16) +declare <16 x i32> @llvm.x86.avx512.mask.valign.d.512(<16 x i32>, <16 x i32>, i32, <16 x i32>, i16) define void @test_mask_store_ss(i8* %ptr, <4 x float> %data, i8 %mask) { - ; CHECK-LABEL: test_mask_store_ss - ; CHECK: vmovss %xmm0, (%rdi) {%k1} ## encoding: [0x62,0xf1,0x7e,0x09,0x11,0x07] +; CHECK-LABEL: test_mask_store_ss: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vmovss %xmm0, (%rdi) {%k1} +; CHECK-NEXT: retq call void @llvm.x86.avx512.mask.store.ss(i8* %ptr, <4 x float> %data, i8 %mask) ret void } @@ -598,15 +999,22 @@ define void @test_mask_store_ss(i8* %ptr, <4 x float> %data, i8 %mask) { declare void @llvm.x86.avx512.mask.store.ss(i8*, <4 x float>, i8 ) define i16 @test_pcmpeq_d(<16 x i32> %a, <16 x i32> %b) { -; CHECK-LABEL: test_pcmpeq_d -; CHECK: vpcmpeqd %zmm1, %zmm0, %k0 ## +; CHECK-LABEL: test_pcmpeq_d: +; CHECK: ## BB#0: +; CHECK-NEXT: vpcmpeqd %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq %res = call i16 @llvm.x86.avx512.mask.pcmpeq.d.512(<16 x i32> %a, <16 x i32> %b, i16 -1) ret i16 %res } define i16 @test_mask_pcmpeq_d(<16 x i32> %a, <16 x i32> %b, i16 %mask) { -; CHECK-LABEL: test_mask_pcmpeq_d -; CHECK: vpcmpeqd %zmm1, %zmm0, %k0 {%k1} ## +; CHECK-LABEL: test_mask_pcmpeq_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpcmpeqd %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq %res = call i16 @llvm.x86.avx512.mask.pcmpeq.d.512(<16 x i32> %a, <16 x i32> %b, i16 %mask) ret i16 %res } @@ -614,15 +1022,23 @@ define i16 @test_mask_pcmpeq_d(<16 x i32> %a, <16 x i32> %b, i16 %mask) { declare i16 @llvm.x86.avx512.mask.pcmpeq.d.512(<16 x i32>, <16 x i32>, i16) define i8 @test_pcmpeq_q(<8 x i64> %a, <8 x i64> %b) { -; CHECK-LABEL: test_pcmpeq_q -; CHECK: vpcmpeqq %zmm1, %zmm0, %k0 ## +; CHECK-LABEL: test_pcmpeq_q: +; CHECK: ## BB#0: +; CHECK-NEXT: vpcmpeqq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq %res = call i8 @llvm.x86.avx512.mask.pcmpeq.q.512(<8 x i64> %a, <8 x i64> %b, i8 -1) ret i8 %res } define i8 @test_mask_pcmpeq_q(<8 x i64> %a, <8 x i64> %b, i8 %mask) { -; CHECK-LABEL: test_mask_pcmpeq_q -; CHECK: vpcmpeqq %zmm1, %zmm0, %k0 {%k1} ## +; CHECK-LABEL: test_mask_pcmpeq_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpcmpeqq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq %res = call i8 @llvm.x86.avx512.mask.pcmpeq.q.512(<8 x i64> %a, <8 x i64> %b, i8 %mask) ret i8 %res } @@ -630,15 +1046,22 @@ define i8 @test_mask_pcmpeq_q(<8 x i64> %a, <8 x i64> %b, i8 %mask) { declare i8 @llvm.x86.avx512.mask.pcmpeq.q.512(<8 x i64>, <8 x i64>, i8) define i16 @test_pcmpgt_d(<16 x i32> %a, <16 x i32> %b) { -; CHECK-LABEL: test_pcmpgt_d -; CHECK: vpcmpgtd %zmm1, %zmm0, %k0 ## +; CHECK-LABEL: test_pcmpgt_d: +; CHECK: ## BB#0: +; CHECK-NEXT: vpcmpgtd %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq %res = call i16 @llvm.x86.avx512.mask.pcmpgt.d.512(<16 x i32> %a, <16 x i32> %b, i16 -1) ret i16 %res } define i16 @test_mask_pcmpgt_d(<16 x i32> %a, <16 x i32> %b, i16 %mask) { -; CHECK-LABEL: test_mask_pcmpgt_d -; CHECK: vpcmpgtd %zmm1, %zmm0, %k0 {%k1} ## +; CHECK-LABEL: test_mask_pcmpgt_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpcmpgtd %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq %res = call i16 @llvm.x86.avx512.mask.pcmpgt.d.512(<16 x i32> %a, <16 x i32> %b, i16 %mask) ret i16 %res } @@ -646,15 +1069,23 @@ define i16 @test_mask_pcmpgt_d(<16 x i32> %a, <16 x i32> %b, i16 %mask) { declare i16 @llvm.x86.avx512.mask.pcmpgt.d.512(<16 x i32>, <16 x i32>, i16) define i8 @test_pcmpgt_q(<8 x i64> %a, <8 x i64> %b) { -; CHECK-LABEL: test_pcmpgt_q -; CHECK: vpcmpgtq %zmm1, %zmm0, %k0 ## +; CHECK-LABEL: test_pcmpgt_q: +; CHECK: ## BB#0: +; CHECK-NEXT: vpcmpgtq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq %res = call i8 @llvm.x86.avx512.mask.pcmpgt.q.512(<8 x i64> %a, <8 x i64> %b, i8 -1) ret i8 %res } define i8 @test_mask_pcmpgt_q(<8 x i64> %a, <8 x i64> %b, i8 %mask) { -; CHECK-LABEL: test_mask_pcmpgt_q -; CHECK: vpcmpgtq %zmm1, %zmm0, %k0 {%k1} ## +; CHECK-LABEL: test_mask_pcmpgt_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpcmpgtq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq %res = call i8 @llvm.x86.avx512.mask.pcmpgt.q.512(<8 x i64> %a, <8 x i64> %b, i8 %mask) ret i8 %res } @@ -662,58 +1093,95 @@ define i8 @test_mask_pcmpgt_q(<8 x i64> %a, <8 x i64> %b, i8 %mask) { declare i8 @llvm.x86.avx512.mask.pcmpgt.q.512(<8 x i64>, <8 x i64>, i8) define <8 x i16> @test_cmp_d_512(<16 x i32> %a0, <16 x i32> %a1) { -; CHECK_LABEL: test_cmp_d_512 -; CHECK: vpcmpeqd %zmm1, %zmm0, %k0 ## +; CHECK-LABEL: test_cmp_d_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpcmpeqd %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %r8d +; CHECK-NEXT: vpcmpltd %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %r9d +; CHECK-NEXT: vpcmpled %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %r10d +; CHECK-NEXT: vpcmpunordd %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %esi +; CHECK-NEXT: vpcmpneqd %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %edi +; CHECK-NEXT: vpcmpnltd %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: vpcmpnled %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %ecx +; CHECK-NEXT: vpcmpordd %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %edx +; CHECK-NEXT: vmovd %r8d, %xmm0 +; CHECK-NEXT: vpinsrw $1, %r9d, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $2, %r10d, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $3, %esi, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $4, %edi, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $5, %eax, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $7, %edx, %xmm0, %xmm0 +; CHECK-NEXT: retq %res0 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 0, i16 -1) %vec0 = insertelement <8 x i16> undef, i16 %res0, i32 0 -; CHECK: vpcmpltd %zmm1, %zmm0, %k0 ## %res1 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 1, i16 -1) %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1 -; CHECK: vpcmpled %zmm1, %zmm0, %k0 ## %res2 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 2, i16 -1) %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2 -; CHECK: vpcmpunordd %zmm1, %zmm0, %k0 ## %res3 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 3, i16 -1) %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3 -; CHECK: vpcmpneqd %zmm1, %zmm0, %k0 ## %res4 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 4, i16 -1) %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4 -; CHECK: vpcmpnltd %zmm1, %zmm0, %k0 ## %res5 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 5, i16 -1) %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5 -; CHECK: vpcmpnled %zmm1, %zmm0, %k0 ## %res6 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 6, i16 -1) %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6 -; CHECK: vpcmpordd %zmm1, %zmm0, %k0 ## %res7 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 7, i16 -1) %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7 ret <8 x i16> %vec7 } define <8 x i16> @test_mask_cmp_d_512(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) { -; CHECK_LABEL: test_mask_cmp_d_512 -; CHECK: vpcmpeqd %zmm1, %zmm0, %k0 {%k1} ## +; CHECK-LABEL: test_mask_cmp_d_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpcmpeqd %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %r8d +; CHECK-NEXT: vpcmpltd %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %r9d +; CHECK-NEXT: vpcmpled %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %r10d +; CHECK-NEXT: vpcmpunordd %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %esi +; CHECK-NEXT: vpcmpneqd %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %edi +; CHECK-NEXT: vpcmpnltd %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: vpcmpnled %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %ecx +; CHECK-NEXT: vpcmpordd %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %edx +; CHECK-NEXT: vmovd %r8d, %xmm0 +; CHECK-NEXT: vpinsrw $1, %r9d, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $2, %r10d, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $3, %esi, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $4, %edi, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $5, %eax, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $7, %edx, %xmm0, %xmm0 +; CHECK-NEXT: retq %res0 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 0, i16 %mask) %vec0 = insertelement <8 x i16> undef, i16 %res0, i32 0 -; CHECK: vpcmpltd %zmm1, %zmm0, %k0 {%k1} ## %res1 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 1, i16 %mask) %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1 -; CHECK: vpcmpled %zmm1, %zmm0, %k0 {%k1} ## %res2 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 2, i16 %mask) %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2 -; CHECK: vpcmpunordd %zmm1, %zmm0, %k0 {%k1} ## %res3 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 3, i16 %mask) %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3 -; CHECK: vpcmpneqd %zmm1, %zmm0, %k0 {%k1} ## %res4 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 4, i16 %mask) %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4 -; CHECK: vpcmpnltd %zmm1, %zmm0, %k0 {%k1} ## %res5 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 5, i16 %mask) %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5 -; CHECK: vpcmpnled %zmm1, %zmm0, %k0 {%k1} ## %res6 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 6, i16 %mask) %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6 -; CHECK: vpcmpordd %zmm1, %zmm0, %k0 {%k1} ## %res7 = call i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 7, i16 %mask) %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7 ret <8 x i16> %vec7 @@ -722,58 +1190,95 @@ define <8 x i16> @test_mask_cmp_d_512(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) declare i16 @llvm.x86.avx512.mask.cmp.d.512(<16 x i32>, <16 x i32>, i32, i16) nounwind readnone define <8 x i16> @test_ucmp_d_512(<16 x i32> %a0, <16 x i32> %a1) { -; CHECK_LABEL: test_ucmp_d_512 -; CHECK: vpcmpequd %zmm1, %zmm0, %k0 ## +; CHECK-LABEL: test_ucmp_d_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpcmpequd %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %r8d +; CHECK-NEXT: vpcmpltud %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %r9d +; CHECK-NEXT: vpcmpleud %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %r10d +; CHECK-NEXT: vpcmpunordud %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %esi +; CHECK-NEXT: vpcmpnequd %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %edi +; CHECK-NEXT: vpcmpnltud %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: vpcmpnleud %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %ecx +; CHECK-NEXT: vpcmpordud %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %edx +; CHECK-NEXT: vmovd %r8d, %xmm0 +; CHECK-NEXT: vpinsrw $1, %r9d, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $2, %r10d, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $3, %esi, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $4, %edi, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $5, %eax, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $7, %edx, %xmm0, %xmm0 +; CHECK-NEXT: retq %res0 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 0, i16 -1) %vec0 = insertelement <8 x i16> undef, i16 %res0, i32 0 -; CHECK: vpcmpltud %zmm1, %zmm0, %k0 ## %res1 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 1, i16 -1) %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1 -; CHECK: vpcmpleud %zmm1, %zmm0, %k0 ## %res2 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 2, i16 -1) %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2 -; CHECK: vpcmpunordud %zmm1, %zmm0, %k0 ## %res3 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 3, i16 -1) %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3 -; CHECK: vpcmpnequd %zmm1, %zmm0, %k0 ## %res4 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 4, i16 -1) %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4 -; CHECK: vpcmpnltud %zmm1, %zmm0, %k0 ## %res5 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 5, i16 -1) %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5 -; CHECK: vpcmpnleud %zmm1, %zmm0, %k0 ## %res6 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 6, i16 -1) %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6 -; CHECK: vpcmpordud %zmm1, %zmm0, %k0 ## %res7 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 7, i16 -1) %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7 ret <8 x i16> %vec7 } define <8 x i16> @test_mask_ucmp_d_512(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) { -; CHECK_LABEL: test_mask_ucmp_d_512 -; CHECK: vpcmpequd %zmm1, %zmm0, %k0 {%k1} ## +; CHECK-LABEL: test_mask_ucmp_d_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpcmpequd %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %r8d +; CHECK-NEXT: vpcmpltud %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %r9d +; CHECK-NEXT: vpcmpleud %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %r10d +; CHECK-NEXT: vpcmpunordud %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %esi +; CHECK-NEXT: vpcmpnequd %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %edi +; CHECK-NEXT: vpcmpnltud %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: vpcmpnleud %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %ecx +; CHECK-NEXT: vpcmpordud %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %edx +; CHECK-NEXT: vmovd %r8d, %xmm0 +; CHECK-NEXT: vpinsrw $1, %r9d, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $2, %r10d, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $3, %esi, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $4, %edi, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $5, %eax, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $7, %edx, %xmm0, %xmm0 +; CHECK-NEXT: retq %res0 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 0, i16 %mask) %vec0 = insertelement <8 x i16> undef, i16 %res0, i32 0 -; CHECK: vpcmpltud %zmm1, %zmm0, %k0 {%k1} ## %res1 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 1, i16 %mask) %vec1 = insertelement <8 x i16> %vec0, i16 %res1, i32 1 -; CHECK: vpcmpleud %zmm1, %zmm0, %k0 {%k1} ## %res2 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 2, i16 %mask) %vec2 = insertelement <8 x i16> %vec1, i16 %res2, i32 2 -; CHECK: vpcmpunordud %zmm1, %zmm0, %k0 {%k1} ## %res3 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 3, i16 %mask) %vec3 = insertelement <8 x i16> %vec2, i16 %res3, i32 3 -; CHECK: vpcmpnequd %zmm1, %zmm0, %k0 {%k1} ## %res4 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 4, i16 %mask) %vec4 = insertelement <8 x i16> %vec3, i16 %res4, i32 4 -; CHECK: vpcmpnltud %zmm1, %zmm0, %k0 {%k1} ## %res5 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 5, i16 %mask) %vec5 = insertelement <8 x i16> %vec4, i16 %res5, i32 5 -; CHECK: vpcmpnleud %zmm1, %zmm0, %k0 {%k1} ## %res6 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 6, i16 %mask) %vec6 = insertelement <8 x i16> %vec5, i16 %res6, i32 6 -; CHECK: vpcmpordud %zmm1, %zmm0, %k0 {%k1} ## %res7 = call i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32> %a0, <16 x i32> %a1, i32 7, i16 %mask) %vec7 = insertelement <8 x i16> %vec6, i16 %res7, i32 7 ret <8 x i16> %vec7 @@ -782,58 +1287,112 @@ define <8 x i16> @test_mask_ucmp_d_512(<16 x i32> %a0, <16 x i32> %a1, i16 %mask declare i16 @llvm.x86.avx512.mask.ucmp.d.512(<16 x i32>, <16 x i32>, i32, i16) nounwind readnone define <8 x i8> @test_cmp_q_512(<8 x i64> %a0, <8 x i64> %a1) { -; CHECK_LABEL: test_cmp_q_512 -; CHECK: vpcmpeqq %zmm1, %zmm0, %k0 ## +; CHECK-LABEL: test_cmp_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpcmpeqq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %r8d +; CHECK-NEXT: vpcmpltq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %r9d +; CHECK-NEXT: vpcmpleq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %r10d +; CHECK-NEXT: vpcmpunordq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %r11d +; CHECK-NEXT: vpcmpneqq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %edi +; CHECK-NEXT: vpcmpnltq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: vpcmpnleq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %ecx +; CHECK-NEXT: vpcmpordq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %edx +; CHECK-NEXT: movzbl %r8b, %esi +; CHECK-NEXT: vpinsrb $0, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %r9b, %esi +; CHECK-NEXT: vpinsrb $2, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %r10b, %esi +; CHECK-NEXT: vpinsrb $4, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %r11b, %esi +; CHECK-NEXT: vpinsrb $6, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %dil, %esi +; CHECK-NEXT: vpinsrb $8, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %al, %eax +; CHECK-NEXT: vpinsrb $10, %eax, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %cl, %eax +; CHECK-NEXT: vpinsrb $12, %eax, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %dl, %eax +; CHECK-NEXT: vpinsrb $14, %eax, %xmm0, %xmm0 +; CHECK-NEXT: retq %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 0, i8 -1) %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0 -; CHECK: vpcmpltq %zmm1, %zmm0, %k0 ## %res1 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 1, i8 -1) %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1 -; CHECK: vpcmpleq %zmm1, %zmm0, %k0 ## %res2 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 2, i8 -1) %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2 -; CHECK: vpcmpunordq %zmm1, %zmm0, %k0 ## %res3 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 3, i8 -1) %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3 -; CHECK: vpcmpneqq %zmm1, %zmm0, %k0 ## %res4 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 4, i8 -1) %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4 -; CHECK: vpcmpnltq %zmm1, %zmm0, %k0 ## %res5 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 5, i8 -1) %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5 -; CHECK: vpcmpnleq %zmm1, %zmm0, %k0 ## %res6 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 6, i8 -1) %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6 -; CHECK: vpcmpordq %zmm1, %zmm0, %k0 ## %res7 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 7, i8 -1) %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7 ret <8 x i8> %vec7 } define <8 x i8> @test_mask_cmp_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) { -; CHECK_LABEL: test_mask_cmp_q_512 -; CHECK: vpcmpeqq %zmm1, %zmm0, %k0 {%k1} ## +; CHECK-LABEL: test_mask_cmp_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpcmpeqq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %r8d +; CHECK-NEXT: vpcmpltq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %r9d +; CHECK-NEXT: vpcmpleq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %r10d +; CHECK-NEXT: vpcmpunordq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %r11d +; CHECK-NEXT: vpcmpneqq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %edi +; CHECK-NEXT: vpcmpnltq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: vpcmpnleq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %ecx +; CHECK-NEXT: vpcmpordq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %edx +; CHECK-NEXT: movzbl %r8b, %esi +; CHECK-NEXT: vpinsrb $0, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %r9b, %esi +; CHECK-NEXT: vpinsrb $2, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %r10b, %esi +; CHECK-NEXT: vpinsrb $4, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %r11b, %esi +; CHECK-NEXT: vpinsrb $6, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %dil, %esi +; CHECK-NEXT: vpinsrb $8, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %al, %eax +; CHECK-NEXT: vpinsrb $10, %eax, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %cl, %eax +; CHECK-NEXT: vpinsrb $12, %eax, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %dl, %eax +; CHECK-NEXT: vpinsrb $14, %eax, %xmm0, %xmm0 +; CHECK-NEXT: retq %res0 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 0, i8 %mask) %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0 -; CHECK: vpcmpltq %zmm1, %zmm0, %k0 {%k1} ## %res1 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 1, i8 %mask) %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1 -; CHECK: vpcmpleq %zmm1, %zmm0, %k0 {%k1} ## %res2 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 2, i8 %mask) %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2 -; CHECK: vpcmpunordq %zmm1, %zmm0, %k0 {%k1} ## %res3 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 3, i8 %mask) %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3 -; CHECK: vpcmpneqq %zmm1, %zmm0, %k0 {%k1} ## %res4 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 4, i8 %mask) %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4 -; CHECK: vpcmpnltq %zmm1, %zmm0, %k0 {%k1} ## %res5 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 5, i8 %mask) %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5 -; CHECK: vpcmpnleq %zmm1, %zmm0, %k0 {%k1} ## %res6 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 6, i8 %mask) %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6 -; CHECK: vpcmpordq %zmm1, %zmm0, %k0 {%k1} ## %res7 = call i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 7, i8 %mask) %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7 ret <8 x i8> %vec7 @@ -842,58 +1401,112 @@ define <8 x i8> @test_mask_cmp_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) { declare i8 @llvm.x86.avx512.mask.cmp.q.512(<8 x i64>, <8 x i64>, i32, i8) nounwind readnone define <8 x i8> @test_ucmp_q_512(<8 x i64> %a0, <8 x i64> %a1) { -; CHECK_LABEL: test_ucmp_q_512 -; CHECK: vpcmpequq %zmm1, %zmm0, %k0 ## +; CHECK-LABEL: test_ucmp_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpcmpequq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %r8d +; CHECK-NEXT: vpcmpltuq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %r9d +; CHECK-NEXT: vpcmpleuq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %r10d +; CHECK-NEXT: vpcmpunorduq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %r11d +; CHECK-NEXT: vpcmpnequq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %edi +; CHECK-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: vpcmpnleuq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %ecx +; CHECK-NEXT: vpcmporduq %zmm1, %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %edx +; CHECK-NEXT: movzbl %r8b, %esi +; CHECK-NEXT: vpinsrb $0, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %r9b, %esi +; CHECK-NEXT: vpinsrb $2, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %r10b, %esi +; CHECK-NEXT: vpinsrb $4, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %r11b, %esi +; CHECK-NEXT: vpinsrb $6, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %dil, %esi +; CHECK-NEXT: vpinsrb $8, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %al, %eax +; CHECK-NEXT: vpinsrb $10, %eax, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %cl, %eax +; CHECK-NEXT: vpinsrb $12, %eax, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %dl, %eax +; CHECK-NEXT: vpinsrb $14, %eax, %xmm0, %xmm0 +; CHECK-NEXT: retq %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 0, i8 -1) %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0 -; CHECK: vpcmpltuq %zmm1, %zmm0, %k0 ## %res1 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 1, i8 -1) %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1 -; CHECK: vpcmpleuq %zmm1, %zmm0, %k0 ## %res2 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 2, i8 -1) %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2 -; CHECK: vpcmpunorduq %zmm1, %zmm0, %k0 ## %res3 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 3, i8 -1) %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3 -; CHECK: vpcmpnequq %zmm1, %zmm0, %k0 ## %res4 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 4, i8 -1) %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4 -; CHECK: vpcmpnltuq %zmm1, %zmm0, %k0 ## %res5 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 5, i8 -1) %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5 -; CHECK: vpcmpnleuq %zmm1, %zmm0, %k0 ## %res6 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 6, i8 -1) %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6 -; CHECK: vpcmporduq %zmm1, %zmm0, %k0 ## %res7 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 7, i8 -1) %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7 ret <8 x i8> %vec7 } define <8 x i8> @test_mask_ucmp_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) { -; CHECK_LABEL: test_mask_ucmp_q_512 -; CHECK: vpcmpequq %zmm1, %zmm0, %k0 {%k1} ## +; CHECK-LABEL: test_mask_ucmp_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpcmpequq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %r8d +; CHECK-NEXT: vpcmpltuq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %r9d +; CHECK-NEXT: vpcmpleuq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %r10d +; CHECK-NEXT: vpcmpunorduq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %r11d +; CHECK-NEXT: vpcmpnequq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %edi +; CHECK-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: vpcmpnleuq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %ecx +; CHECK-NEXT: vpcmporduq %zmm1, %zmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %edx +; CHECK-NEXT: movzbl %r8b, %esi +; CHECK-NEXT: vpinsrb $0, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %r9b, %esi +; CHECK-NEXT: vpinsrb $2, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %r10b, %esi +; CHECK-NEXT: vpinsrb $4, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %r11b, %esi +; CHECK-NEXT: vpinsrb $6, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %dil, %esi +; CHECK-NEXT: vpinsrb $8, %esi, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %al, %eax +; CHECK-NEXT: vpinsrb $10, %eax, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %cl, %eax +; CHECK-NEXT: vpinsrb $12, %eax, %xmm0, %xmm0 +; CHECK-NEXT: movzbl %dl, %eax +; CHECK-NEXT: vpinsrb $14, %eax, %xmm0, %xmm0 +; CHECK-NEXT: retq %res0 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 0, i8 %mask) %vec0 = insertelement <8 x i8> undef, i8 %res0, i32 0 -; CHECK: vpcmpltuq %zmm1, %zmm0, %k0 {%k1} ## %res1 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 1, i8 %mask) %vec1 = insertelement <8 x i8> %vec0, i8 %res1, i32 1 -; CHECK: vpcmpleuq %zmm1, %zmm0, %k0 {%k1} ## %res2 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 2, i8 %mask) %vec2 = insertelement <8 x i8> %vec1, i8 %res2, i32 2 -; CHECK: vpcmpunorduq %zmm1, %zmm0, %k0 {%k1} ## %res3 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 3, i8 %mask) %vec3 = insertelement <8 x i8> %vec2, i8 %res3, i32 3 -; CHECK: vpcmpnequq %zmm1, %zmm0, %k0 {%k1} ## %res4 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 4, i8 %mask) %vec4 = insertelement <8 x i8> %vec3, i8 %res4, i32 4 -; CHECK: vpcmpnltuq %zmm1, %zmm0, %k0 {%k1} ## %res5 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 5, i8 %mask) %vec5 = insertelement <8 x i8> %vec4, i8 %res5, i32 5 -; CHECK: vpcmpnleuq %zmm1, %zmm0, %k0 {%k1} ## %res6 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 6, i8 %mask) %vec6 = insertelement <8 x i8> %vec5, i8 %res6, i32 6 -; CHECK: vpcmporduq %zmm1, %zmm0, %k0 {%k1} ## %res7 = call i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64> %a0, <8 x i64> %a1, i32 7, i8 %mask) %vec7 = insertelement <8 x i8> %vec6, i8 %res7, i32 7 ret <8 x i8> %vec7 @@ -903,57 +1516,77 @@ declare i8 @llvm.x86.avx512.mask.ucmp.q.512(<8 x i64>, <8 x i64>, i32, i8) nounw define <4 x float> @test_mask_vextractf32x4(<4 x float> %b, <16 x float> %a, i8 %mask) { ; CHECK-LABEL: test_mask_vextractf32x4: -; CHECK: vextractf32x4 $2, %zmm1, %xmm0 {%k1} - %res = call <4 x float> @llvm.x86.avx512.mask.vextractf32x4.512(<16 x float> %a, i8 2, <4 x float> %b, i8 %mask) +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vextractf32x4 $2, %zmm1, %xmm0 {%k1} +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.vextractf32x4.512(<16 x float> %a, i32 2, <4 x float> %b, i8 %mask) ret <4 x float> %res } -declare <4 x float> @llvm.x86.avx512.mask.vextractf32x4.512(<16 x float>, i8, <4 x float>, i8) +declare <4 x float> @llvm.x86.avx512.mask.vextractf32x4.512(<16 x float>, i32, <4 x float>, i8) define <4 x i64> @test_mask_vextracti64x4(<4 x i64> %b, <8 x i64> %a, i8 %mask) { ; CHECK-LABEL: test_mask_vextracti64x4: -; CHECK: vextracti64x4 $2, %zmm1, %ymm0 {%k1} - %res = call <4 x i64> @llvm.x86.avx512.mask.vextracti64x4.512(<8 x i64> %a, i8 2, <4 x i64> %b, i8 %mask) +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vextracti64x4 $2, %zmm1, %ymm0 {%k1} +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.mask.vextracti64x4.512(<8 x i64> %a, i32 2, <4 x i64> %b, i8 %mask) ret <4 x i64> %res } -declare <4 x i64> @llvm.x86.avx512.mask.vextracti64x4.512(<8 x i64>, i8, <4 x i64>, i8) +declare <4 x i64> @llvm.x86.avx512.mask.vextracti64x4.512(<8 x i64>, i32, <4 x i64>, i8) define <4 x i32> @test_maskz_vextracti32x4(<16 x i32> %a, i8 %mask) { ; CHECK-LABEL: test_maskz_vextracti32x4: -; CHECK: vextracti32x4 $2, %zmm0, %xmm0 {%k1} {z} - %res = call <4 x i32> @llvm.x86.avx512.mask.vextracti32x4.512(<16 x i32> %a, i8 2, <4 x i32> zeroinitializer, i8 %mask) +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vextracti32x4 $2, %zmm0, %xmm0 {%k1} {z} +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.vextracti32x4.512(<16 x i32> %a, i32 2, <4 x i32> zeroinitializer, i8 %mask) ret <4 x i32> %res } -declare <4 x i32> @llvm.x86.avx512.mask.vextracti32x4.512(<16 x i32>, i8, <4 x i32>, i8) +declare <4 x i32> @llvm.x86.avx512.mask.vextracti32x4.512(<16 x i32>, i32, <4 x i32>, i8) define <4 x double> @test_vextractf64x4(<8 x double> %a) { ; CHECK-LABEL: test_vextractf64x4: -; CHECK: vextractf64x4 $2, %zmm0, %ymm0 ## - %res = call <4 x double> @llvm.x86.avx512.mask.vextractf64x4.512(<8 x double> %a, i8 2, <4 x double> zeroinitializer, i8 -1) +; CHECK: ## BB#0: +; CHECK-NEXT: vextractf64x4 $2, %zmm0, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.avx512.mask.vextractf64x4.512(<8 x double> %a, i32 2, <4 x double> zeroinitializer, i8 -1) ret <4 x double> %res } -declare <4 x double> @llvm.x86.avx512.mask.vextractf64x4.512(<8 x double>, i8, <4 x double>, i8) +declare <4 x double> @llvm.x86.avx512.mask.vextractf64x4.512(<8 x double>, i32, <4 x double>, i8) define <16 x i32> @test_x86_avx512_pslli_d(<16 x i32> %a0) { - ; CHECK-LABEL: test_x86_avx512_pslli_d - ; CHECK: vpslld +; CHECK-LABEL: test_x86_avx512_pslli_d: +; CHECK: ## BB#0: +; CHECK-NEXT: vpslld $7, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.pslli.d(<16 x i32> %a0, i32 7, <16 x i32> zeroinitializer, i16 -1) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_mask_pslli_d(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_pslli_d - ; CHECK: vpslld $7, %zmm0, %zmm1 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_pslli_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpslld $7, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.pslli.d(<16 x i32> %a0, i32 7, <16 x i32> %a1, i16 %mask) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_maskz_pslli_d(<16 x i32> %a0, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_pslli_d - ; CHECK: vpslld $7, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_pslli_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpslld $7, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.pslli.d(<16 x i32> %a0, i32 7, <16 x i32> zeroinitializer, i16 %mask) ret <16 x i32> %res } @@ -961,22 +1594,33 @@ define <16 x i32> @test_x86_avx512_maskz_pslli_d(<16 x i32> %a0, i16 %mask) { declare <16 x i32> @llvm.x86.avx512.mask.pslli.d(<16 x i32>, i32, <16 x i32>, i16) nounwind readnone define <8 x i64> @test_x86_avx512_pslli_q(<8 x i64> %a0) { - ; CHECK-LABEL: test_x86_avx512_pslli_q - ; CHECK: vpsllq +; CHECK-LABEL: test_x86_avx512_pslli_q: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsllq $7, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pslli.q(<8 x i64> %a0, i32 7, <8 x i64> zeroinitializer, i8 -1) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_mask_pslli_q(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_pslli_q - ; CHECK: vpsllq $7, %zmm0, %zmm1 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_pslli_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsllq $7, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pslli.q(<8 x i64> %a0, i32 7, <8 x i64> %a1, i8 %mask) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_maskz_pslli_q(<8 x i64> %a0, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_pslli_q - ; CHECK: vpsllq $7, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_pslli_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsllq $7, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pslli.q(<8 x i64> %a0, i32 7, <8 x i64> zeroinitializer, i8 %mask) ret <8 x i64> %res } @@ -984,22 +1628,31 @@ define <8 x i64> @test_x86_avx512_maskz_pslli_q(<8 x i64> %a0, i8 %mask) { declare <8 x i64> @llvm.x86.avx512.mask.pslli.q(<8 x i64>, i32, <8 x i64>, i8) nounwind readnone define <16 x i32> @test_x86_avx512_psrli_d(<16 x i32> %a0) { - ; CHECK-LABEL: test_x86_avx512_psrli_d - ; CHECK: vpsrld +; CHECK-LABEL: test_x86_avx512_psrli_d: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsrld $7, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psrli.d(<16 x i32> %a0, i32 7, <16 x i32> zeroinitializer, i16 -1) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_mask_psrli_d(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_psrli_d - ; CHECK: vpsrld $7, %zmm0, %zmm1 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_psrli_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpsrld $7, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psrli.d(<16 x i32> %a0, i32 7, <16 x i32> %a1, i16 %mask) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_maskz_psrli_d(<16 x i32> %a0, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_psrli_d - ; CHECK: vpsrld $7, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_psrli_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpsrld $7, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psrli.d(<16 x i32> %a0, i32 7, <16 x i32> zeroinitializer, i16 %mask) ret <16 x i32> %res } @@ -1007,22 +1660,33 @@ define <16 x i32> @test_x86_avx512_maskz_psrli_d(<16 x i32> %a0, i16 %mask) { declare <16 x i32> @llvm.x86.avx512.mask.psrli.d(<16 x i32>, i32, <16 x i32>, i16) nounwind readnone define <8 x i64> @test_x86_avx512_psrli_q(<8 x i64> %a0) { - ; CHECK-LABEL: test_x86_avx512_psrli_q - ; CHECK: vpsrlq - %res = call <8 x i64> @llvm.x86.avx512.mask.psrli.q(<8 x i64> %a0, i32 7, <8 x i64> zeroinitializer, i8 -1) +; CHECK-LABEL: test_x86_avx512_psrli_q: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsrlq $7, %zmm0, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.psrli.q(<8 x i64> %a0, i32 7, <8 x i64> zeroinitializer, i8 -1) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_mask_psrli_q(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_psrli_q - ; CHECK: vpsrlq $7, %zmm0, %zmm1 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_psrli_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsrlq $7, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psrli.q(<8 x i64> %a0, i32 7, <8 x i64> %a1, i8 %mask) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_maskz_psrli_q(<8 x i64> %a0, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_psrli_q - ; CHECK: vpsrlq $7, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_psrli_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsrlq $7, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psrli.q(<8 x i64> %a0, i32 7, <8 x i64> zeroinitializer, i8 %mask) ret <8 x i64> %res } @@ -1030,22 +1694,31 @@ define <8 x i64> @test_x86_avx512_maskz_psrli_q(<8 x i64> %a0, i8 %mask) { declare <8 x i64> @llvm.x86.avx512.mask.psrli.q(<8 x i64>, i32, <8 x i64>, i8) nounwind readnone define <16 x i32> @test_x86_avx512_psrai_d(<16 x i32> %a0) { - ; CHECK-LABEL: test_x86_avx512_psrai_d - ; CHECK: vpsrad +; CHECK-LABEL: test_x86_avx512_psrai_d: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsrad $7, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psrai.d(<16 x i32> %a0, i32 7, <16 x i32> zeroinitializer, i16 -1) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_mask_psrai_d(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_psrai_d - ; CHECK: vpsrad $7, %zmm0, %zmm1 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_psrai_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpsrad $7, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psrai.d(<16 x i32> %a0, i32 7, <16 x i32> %a1, i16 %mask) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_maskz_psrai_d(<16 x i32> %a0, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_psrai_d - ; CHECK: vpsrad $7, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_psrai_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpsrad $7, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psrai.d(<16 x i32> %a0, i32 7, <16 x i32> zeroinitializer, i16 %mask) ret <16 x i32> %res } @@ -1053,22 +1726,33 @@ define <16 x i32> @test_x86_avx512_maskz_psrai_d(<16 x i32> %a0, i16 %mask) { declare <16 x i32> @llvm.x86.avx512.mask.psrai.d(<16 x i32>, i32, <16 x i32>, i16) nounwind readnone define <8 x i64> @test_x86_avx512_psrai_q(<8 x i64> %a0) { - ; CHECK-LABEL: test_x86_avx512_psrai_q - ; CHECK: vpsraq +; CHECK-LABEL: test_x86_avx512_psrai_q: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsraq $7, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psrai.q(<8 x i64> %a0, i32 7, <8 x i64> zeroinitializer, i8 -1) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_mask_psrai_q(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_psrai_q - ; CHECK: vpsraq $7, %zmm0, %zmm1 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_psrai_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsraq $7, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psrai.q(<8 x i64> %a0, i32 7, <8 x i64> %a1, i8 %mask) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_maskz_psrai_q(<8 x i64> %a0, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_psrai_q - ; CHECK: vpsraq $7, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_psrai_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsraq $7, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psrai.q(<8 x i64> %a0, i32 7, <8 x i64> zeroinitializer, i8 %mask) ret <8 x i64> %res } @@ -1076,22 +1760,31 @@ define <8 x i64> @test_x86_avx512_maskz_psrai_q(<8 x i64> %a0, i8 %mask) { declare <8 x i64> @llvm.x86.avx512.mask.psrai.q(<8 x i64>, i32, <8 x i64>, i8) nounwind readnone define <16 x i32> @test_x86_avx512_psll_d(<16 x i32> %a0, <4 x i32> %a1) { - ; CHECK-LABEL: test_x86_avx512_psll_d - ; CHECK: vpslld +; CHECK-LABEL: test_x86_avx512_psll_d: +; CHECK: ## BB#0: +; CHECK-NEXT: vpslld %xmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psll.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> zeroinitializer, i16 -1) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_mask_psll_d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %a2, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_psll_d - ; CHECK: vpslld %xmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_psll_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpslld %xmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psll.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %a2, i16 %mask) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_maskz_psll_d(<16 x i32> %a0, <4 x i32> %a1, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_psll_d - ; CHECK: vpslld %xmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_psll_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpslld %xmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psll.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> zeroinitializer, i16 %mask) ret <16 x i32> %res } @@ -1099,22 +1792,33 @@ define <16 x i32> @test_x86_avx512_maskz_psll_d(<16 x i32> %a0, <4 x i32> %a1, i declare <16 x i32> @llvm.x86.avx512.mask.psll.d(<16 x i32>, <4 x i32>, <16 x i32>, i16) nounwind readnone define <8 x i64> @test_x86_avx512_psll_q(<8 x i64> %a0, <2 x i64> %a1) { - ; CHECK-LABEL: test_x86_avx512_psll_q - ; CHECK: vpsllq +; CHECK-LABEL: test_x86_avx512_psll_q: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsllq %xmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psll.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> zeroinitializer, i8 -1) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_mask_psll_q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %a2, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_psll_q - ; CHECK: vpsllq %xmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_psll_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsllq %xmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psll.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %a2, i8 %mask) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_maskz_psll_q(<8 x i64> %a0, <2 x i64> %a1, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_psll_q - ; CHECK: vpsllq %xmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_psll_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsllq %xmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psll.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> zeroinitializer, i8 %mask) ret <8 x i64> %res } @@ -1122,22 +1826,31 @@ define <8 x i64> @test_x86_avx512_maskz_psll_q(<8 x i64> %a0, <2 x i64> %a1, i8 declare <8 x i64> @llvm.x86.avx512.mask.psll.q(<8 x i64>, <2 x i64>, <8 x i64>, i8) nounwind readnone define <16 x i32> @test_x86_avx512_psrl_d(<16 x i32> %a0, <4 x i32> %a1) { - ; CHECK-LABEL: test_x86_avx512_psrl_d - ; CHECK: vpsrld +; CHECK-LABEL: test_x86_avx512_psrl_d: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsrld %xmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psrl.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> zeroinitializer, i16 -1) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_mask_psrl_d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %a2, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_psrl_d - ; CHECK: vpsrld %xmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_psrl_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpsrld %xmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psrl.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %a2, i16 %mask) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_maskz_psrl_d(<16 x i32> %a0, <4 x i32> %a1, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_psrl_d - ; CHECK: vpsrld %xmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_psrl_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpsrld %xmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psrl.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> zeroinitializer, i16 %mask) ret <16 x i32> %res } @@ -1145,22 +1858,33 @@ define <16 x i32> @test_x86_avx512_maskz_psrl_d(<16 x i32> %a0, <4 x i32> %a1, i declare <16 x i32> @llvm.x86.avx512.mask.psrl.d(<16 x i32>, <4 x i32>, <16 x i32>, i16) nounwind readnone define <8 x i64> @test_x86_avx512_psrl_q(<8 x i64> %a0, <2 x i64> %a1) { - ; CHECK-LABEL: test_x86_avx512_psrl_q - ; CHECK: vpsrlq +; CHECK-LABEL: test_x86_avx512_psrl_q: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsrlq %xmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psrl.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> zeroinitializer, i8 -1) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_mask_psrl_q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %a2, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_psrl_q - ; CHECK: vpsrlq %xmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_psrl_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsrlq %xmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psrl.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %a2, i8 %mask) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_maskz_psrl_q(<8 x i64> %a0, <2 x i64> %a1, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_psrl_q - ; CHECK: vpsrlq %xmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_psrl_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsrlq %xmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psrl.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> zeroinitializer, i8 %mask) ret <8 x i64> %res } @@ -1168,22 +1892,31 @@ define <8 x i64> @test_x86_avx512_maskz_psrl_q(<8 x i64> %a0, <2 x i64> %a1, i8 declare <8 x i64> @llvm.x86.avx512.mask.psrl.q(<8 x i64>, <2 x i64>, <8 x i64>, i8) nounwind readnone define <16 x i32> @test_x86_avx512_psra_d(<16 x i32> %a0, <4 x i32> %a1) { - ; CHECK-LABEL: test_x86_avx512_psra_d - ; CHECK: vpsrad +; CHECK-LABEL: test_x86_avx512_psra_d: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsrad %xmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psra.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> zeroinitializer, i16 -1) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_mask_psra_d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %a2, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_psra_d - ; CHECK: vpsrad %xmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_psra_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpsrad %xmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psra.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %a2, i16 %mask) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_maskz_psra_d(<16 x i32> %a0, <4 x i32> %a1, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_psra_d - ; CHECK: vpsrad %xmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_psra_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpsrad %xmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psra.d(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> zeroinitializer, i16 %mask) ret <16 x i32> %res } @@ -1191,22 +1924,33 @@ define <16 x i32> @test_x86_avx512_maskz_psra_d(<16 x i32> %a0, <4 x i32> %a1, i declare <16 x i32> @llvm.x86.avx512.mask.psra.d(<16 x i32>, <4 x i32>, <16 x i32>, i16) nounwind readnone define <8 x i64> @test_x86_avx512_psra_q(<8 x i64> %a0, <2 x i64> %a1) { - ; CHECK-LABEL: test_x86_avx512_psra_q - ; CHECK: vpsraq +; CHECK-LABEL: test_x86_avx512_psra_q: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsraq %xmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psra.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> zeroinitializer, i8 -1) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_mask_psra_q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %a2, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_psra_q - ; CHECK: vpsraq %xmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_psra_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsraq %xmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psra.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %a2, i8 %mask) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_maskz_psra_q(<8 x i64> %a0, <2 x i64> %a1, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_psra_q - ; CHECK: vpsraq %xmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_psra_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsraq %xmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psra.q(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> zeroinitializer, i8 %mask) ret <8 x i64> %res } @@ -1214,22 +1958,31 @@ define <8 x i64> @test_x86_avx512_maskz_psra_q(<8 x i64> %a0, <2 x i64> %a1, i8 declare <8 x i64> @llvm.x86.avx512.mask.psra.q(<8 x i64>, <2 x i64>, <8 x i64>, i8) nounwind readnone define <16 x i32> @test_x86_avx512_psllv_d(<16 x i32> %a0, <16 x i32> %a1) { - ; CHECK-LABEL: test_x86_avx512_psllv_d - ; CHECK: vpsllvd +; CHECK-LABEL: test_x86_avx512_psllv_d: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsllvd %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psllv.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> zeroinitializer, i16 -1) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_mask_psllv_d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_psllv_d - ; CHECK: vpsllvd %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_psllv_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpsllvd %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psllv.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_maskz_psllv_d(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_psllv_d - ; CHECK: vpsllvd %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_psllv_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpsllvd %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psllv.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> zeroinitializer, i16 %mask) ret <16 x i32> %res } @@ -1237,22 +1990,33 @@ define <16 x i32> @test_x86_avx512_maskz_psllv_d(<16 x i32> %a0, <16 x i32> %a1, declare <16 x i32> @llvm.x86.avx512.mask.psllv.d(<16 x i32>, <16 x i32>, <16 x i32>, i16) nounwind readnone define <8 x i64> @test_x86_avx512_psllv_q(<8 x i64> %a0, <8 x i64> %a1) { - ; CHECK-LABEL: test_x86_avx512_psllv_q - ; CHECK: vpsllvq +; CHECK-LABEL: test_x86_avx512_psllv_q: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsllvq %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psllv.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> zeroinitializer, i8 -1) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_mask_psllv_q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_psllv_q - ; CHECK: vpsllvq %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_psllv_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsllvq %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psllv.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_maskz_psllv_q(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_psllv_q - ; CHECK: vpsllvq %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_psllv_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsllvq %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psllv.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> zeroinitializer, i8 %mask) ret <8 x i64> %res } @@ -1261,22 +2025,31 @@ declare <8 x i64> @llvm.x86.avx512.mask.psllv.q(<8 x i64>, <8 x i64>, <8 x i64>, define <16 x i32> @test_x86_avx512_psrav_d(<16 x i32> %a0, <16 x i32> %a1) { - ; CHECK-LABEL: test_x86_avx512_psrav_d - ; CHECK: vpsravd +; CHECK-LABEL: test_x86_avx512_psrav_d: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsravd %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psrav.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> zeroinitializer, i16 -1) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_mask_psrav_d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_psrav_d - ; CHECK: vpsravd %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_psrav_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpsravd %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psrav.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_maskz_psrav_d(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_psrav_d - ; CHECK: vpsravd %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_psrav_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpsravd %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psrav.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> zeroinitializer, i16 %mask) ret <16 x i32> %res } @@ -1284,22 +2057,33 @@ define <16 x i32> @test_x86_avx512_maskz_psrav_d(<16 x i32> %a0, <16 x i32> %a1, declare <16 x i32> @llvm.x86.avx512.mask.psrav.d(<16 x i32>, <16 x i32>, <16 x i32>, i16) nounwind readnone define <8 x i64> @test_x86_avx512_psrav_q(<8 x i64> %a0, <8 x i64> %a1) { - ; CHECK-LABEL: test_x86_avx512_psrav_q - ; CHECK: vpsravq +; CHECK-LABEL: test_x86_avx512_psrav_q: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsravq %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psrav.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> zeroinitializer, i8 -1) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_mask_psrav_q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_psrav_q - ; CHECK: vpsravq %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_psrav_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsravq %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psrav.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_maskz_psrav_q(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_psrav_q - ; CHECK: vpsravq %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_psrav_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsravq %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psrav.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> zeroinitializer, i8 %mask) ret <8 x i64> %res } @@ -1307,22 +2091,31 @@ define <8 x i64> @test_x86_avx512_maskz_psrav_q(<8 x i64> %a0, <8 x i64> %a1, i8 declare <8 x i64> @llvm.x86.avx512.mask.psrav.q(<8 x i64>, <8 x i64>, <8 x i64>, i8) nounwind readnone define <16 x i32> @test_x86_avx512_psrlv_d(<16 x i32> %a0, <16 x i32> %a1) { - ; CHECK-LABEL: test_x86_avx512_psrlv_d - ; CHECK: vpsrlvd +; CHECK-LABEL: test_x86_avx512_psrlv_d: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsrlvd %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psrlv.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> zeroinitializer, i16 -1) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_mask_psrlv_d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_psrlv_d - ; CHECK: vpsrlvd %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_psrlv_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpsrlvd %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psrlv.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask) ret <16 x i32> %res } define <16 x i32> @test_x86_avx512_maskz_psrlv_d(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_psrlv_d - ; CHECK: vpsrlvd %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_psrlv_d: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpsrlvd %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psrlv.d(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> zeroinitializer, i16 %mask) ret <16 x i32> %res } @@ -1330,22 +2123,33 @@ define <16 x i32> @test_x86_avx512_maskz_psrlv_d(<16 x i32> %a0, <16 x i32> %a1, declare <16 x i32> @llvm.x86.avx512.mask.psrlv.d(<16 x i32>, <16 x i32>, <16 x i32>, i16) nounwind readnone define <8 x i64> @test_x86_avx512_psrlv_q(<8 x i64> %a0, <8 x i64> %a1) { - ; CHECK-LABEL: test_x86_avx512_psrlv_q - ; CHECK: vpsrlvq +; CHECK-LABEL: test_x86_avx512_psrlv_q: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsrlvq %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psrlv.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> zeroinitializer, i8 -1) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_mask_psrlv_q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_mask_psrlv_q - ; CHECK: vpsrlvq %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_x86_avx512_mask_psrlv_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsrlvq %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psrlv.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask) ret <8 x i64> %res } define <8 x i64> @test_x86_avx512_maskz_psrlv_q(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) { - ; CHECK-LABEL: test_x86_avx512_maskz_psrlv_q - ; CHECK: vpsrlvq %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_x86_avx512_maskz_psrlv_q: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsrlvq %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psrlv.q(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> zeroinitializer, i8 %mask) ret <8 x i64> %res } @@ -1353,8 +2157,10 @@ define <8 x i64> @test_x86_avx512_maskz_psrlv_q(<8 x i64> %a0, <8 x i64> %a1, i8 declare <8 x i64> @llvm.x86.avx512.mask.psrlv.q(<8 x i64>, <8 x i64>, <8 x i64>, i8) nounwind readnone define <8 x i64> @test_x86_avx512_psrlv_q_memop(<8 x i64> %a0, <8 x i64>* %ptr) { - ; CHECK-LABEL: test_x86_avx512_psrlv_q_memop - ; CHECK: vpsrlvq (% +; CHECK-LABEL: test_x86_avx512_psrlv_q_memop: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsrlvq (%rdi), %zmm0, %zmm0 +; CHECK-NEXT: retq %b = load <8 x i64>, <8 x i64>* %ptr %res = call <8 x i64> @llvm.x86.avx512.mask.psrlv.q(<8 x i64> %a0, <8 x i64> %b, <8 x i64> zeroinitializer, i8 -1) ret <8 x i64> %res @@ -1365,64 +2171,80 @@ declare <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float>, <16 x float> declare <8 x double> @llvm.x86.avx512.mask.mul.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32) define <16 x float> @test_vsubps_rn(<16 x float> %a0, <16 x float> %a1) { - ; CHECK-LABEL: test_vsubps_rn - ; CHECK: vsubps {rn-sae}{{.*}} ## encoding: [0x62,0xf1,0x7c,0x18,0x5c,0xc1] +; CHECK-LABEL: test_vsubps_rn: +; CHECK: ## BB#0: +; CHECK-NEXT: vsubps {rn-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> zeroinitializer, i16 -1, i32 0) ret <16 x float> %res } define <16 x float> @test_vsubps_rd(<16 x float> %a0, <16 x float> %a1) { - ; CHECK-LABEL: test_vsubps_rd - ; CHECK: vsubps {rd-sae}{{.*}} ## encoding: [0x62,0xf1,0x7c,0x38,0x5c,0xc1] +; CHECK-LABEL: test_vsubps_rd: +; CHECK: ## BB#0: +; CHECK-NEXT: vsubps {rd-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> zeroinitializer, i16 -1, i32 1) ret <16 x float> %res } define <16 x float> @test_vsubps_ru(<16 x float> %a0, <16 x float> %a1) { - ; CHECK-LABEL: test_vsubps_ru - ; CHECK: vsubps {ru-sae}{{.*}} ## encoding: [0x62,0xf1,0x7c,0x58,0x5c,0xc1] +; CHECK-LABEL: test_vsubps_ru: +; CHECK: ## BB#0: +; CHECK-NEXT: vsubps {ru-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> zeroinitializer, i16 -1, i32 2) ret <16 x float> %res } define <16 x float> @test_vsubps_rz(<16 x float> %a0, <16 x float> %a1) { - ; CHECK-LABEL: test_vsubps_rz - ; CHECK: vsubps {rz-sae}{{.*}} ## encoding: [0x62,0xf1,0x7c,0x78,0x5c,0xc1] +; CHECK-LABEL: test_vsubps_rz: +; CHECK: ## BB#0: +; CHECK-NEXT: vsubps {rz-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> zeroinitializer, i16 -1, i32 3) ret <16 x float> %res } define <16 x float> @test_vmulps_rn(<16 x float> %a0, <16 x float> %a1) { - ; CHECK-LABEL: test_vmulps_rn - ; CHECK: vmulps {rn-sae}{{.*}} ## encoding: [0x62,0xf1,0x7c,0x18,0x59,0xc1] +; CHECK-LABEL: test_vmulps_rn: +; CHECK: ## BB#0: +; CHECK-NEXT: vmulps {rn-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> zeroinitializer, i16 -1, i32 0) ret <16 x float> %res } define <16 x float> @test_vmulps_rd(<16 x float> %a0, <16 x float> %a1) { - ; CHECK-LABEL: test_vmulps_rd - ; CHECK: vmulps {rd-sae}{{.*}} ## encoding: [0x62,0xf1,0x7c,0x38,0x59,0xc1] +; CHECK-LABEL: test_vmulps_rd: +; CHECK: ## BB#0: +; CHECK-NEXT: vmulps {rd-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> zeroinitializer, i16 -1, i32 1) ret <16 x float> %res } define <16 x float> @test_vmulps_ru(<16 x float> %a0, <16 x float> %a1) { - ; CHECK-LABEL: test_vmulps_ru - ; CHECK: vmulps {ru-sae}{{.*}} ## encoding: [0x62,0xf1,0x7c,0x58,0x59,0xc1] +; CHECK-LABEL: test_vmulps_ru: +; CHECK: ## BB#0: +; CHECK-NEXT: vmulps {ru-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> zeroinitializer, i16 -1, i32 2) ret <16 x float> %res } define <16 x float> @test_vmulps_rz(<16 x float> %a0, <16 x float> %a1) { - ; CHECK-LABEL: test_vmulps_rz - ; CHECK: vmulps {rz-sae}{{.*}} ## encoding: [0x62,0xf1,0x7c,0x78,0x59,0xc1] +; CHECK-LABEL: test_vmulps_rz: +; CHECK: ## BB#0: +; CHECK-NEXT: vmulps {rz-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> zeroinitializer, i16 -1, i32 3) ret <16 x float> %res @@ -1430,32 +2252,44 @@ define <16 x float> @test_vmulps_rz(<16 x float> %a0, <16 x float> %a1) { ;; mask float define <16 x float> @test_vmulps_mask_rn(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ; CHECK-LABEL: test_vmulps_mask_rn - ; CHECK: vmulps {rn-sae}{{.*}}{%k1} {z} ## encoding: [0x62,0xf1,0x7c,0x99,0x59,0xc1] +; CHECK-LABEL: test_vmulps_mask_rn: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmulps {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> zeroinitializer, i16 %mask, i32 0) ret <16 x float> %res } define <16 x float> @test_vmulps_mask_rd(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ; CHECK-LABEL: test_vmulps_mask_rd - ; CHECK: vmulps {rd-sae}{{.*}}{%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xb9,0x59,0xc1] +; CHECK-LABEL: test_vmulps_mask_rd: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmulps {rd-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> zeroinitializer, i16 %mask, i32 1) ret <16 x float> %res } define <16 x float> @test_vmulps_mask_ru(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ; CHECK-LABEL: test_vmulps_mask_ru - ; CHECK: vmulps {ru-sae}{{.*}}{%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xd9,0x59,0xc1] +; CHECK-LABEL: test_vmulps_mask_ru: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmulps {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> zeroinitializer, i16 %mask, i32 2) ret <16 x float> %res } define <16 x float> @test_vmulps_mask_rz(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ; CHECK-LABEL: test_vmulps_mask_rz - ; CHECK: vmulps {rz-sae}{{.*}}{%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xf9,0x59,0xc1] +; CHECK-LABEL: test_vmulps_mask_rz: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmulps {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> zeroinitializer, i16 %mask, i32 3) ret <16 x float> %res @@ -1463,32 +2297,48 @@ define <16 x float> @test_vmulps_mask_rz(<16 x float> %a0, <16 x float> %a1, i16 ;; With Passthru value define <16 x float> @test_vmulps_mask_passthru_rn(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask) { - ; CHECK-LABEL: test_vmulps_mask_passthru_rn - ; CHECK: vmulps {rn-sae}{{.*}}{%k1} ## encoding: [0x62,0xf1,0x7c,0x19,0x59,0xd1] +; CHECK-LABEL: test_vmulps_mask_passthru_rn: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmulps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask, i32 0) ret <16 x float> %res } define <16 x float> @test_vmulps_mask_passthru_rd(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask) { - ; CHECK-LABEL: test_vmulps_mask_passthru_rd - ; CHECK: vmulps {rd-sae}{{.*}}{%k1} ## encoding: [0x62,0xf1,0x7c,0x39,0x59,0xd1] +; CHECK-LABEL: test_vmulps_mask_passthru_rd: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmulps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask, i32 1) ret <16 x float> %res } define <16 x float> @test_vmulps_mask_passthru_ru(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask) { - ; CHECK-LABEL: test_vmulps_mask_passthru_ru - ; CHECK: vmulps {ru-sae}{{.*}}{%k1} ## encoding: [0x62,0xf1,0x7c,0x59,0x59,0xd1] +; CHECK-LABEL: test_vmulps_mask_passthru_ru: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmulps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask, i32 2) ret <16 x float> %res } define <16 x float> @test_vmulps_mask_passthru_rz(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask) { - ; CHECK-LABEL: test_vmulps_mask_passthru_rz - ; CHECK: vmulps {rz-sae}{{.*}}{%k1} ## encoding: [0x62,0xf1,0x7c,0x79,0x59,0xd1] +; CHECK-LABEL: test_vmulps_mask_passthru_rz: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmulps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.mul.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask, i32 3) ret <16 x float> %res @@ -1496,47 +2346,69 @@ define <16 x float> @test_vmulps_mask_passthru_rz(<16 x float> %a0, <16 x float> ;; mask double define <8 x double> @test_vmulpd_mask_rn(<8 x double> %a0, <8 x double> %a1, i8 %mask) { - ; CHECK-LABEL: test_vmulpd_mask_rn - ; CHECK: vmulpd {rn-sae}{{.*}}{%k1} {z} ## encoding: [0x62,0xf1,0xfd,0x99,0x59,0xc1] +; CHECK-LABEL: test_vmulpd_mask_rn: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmulpd {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <8 x double> @llvm.x86.avx512.mask.mul.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> zeroinitializer, i8 %mask, i32 0) ret <8 x double> %res } define <8 x double> @test_vmulpd_mask_rd(<8 x double> %a0, <8 x double> %a1, i8 %mask) { - ; CHECK-LABEL: test_vmulpd_mask_rd - ; CHECK: vmulpd {rd-sae}{{.*}}{%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xb9,0x59,0xc1] +; CHECK-LABEL: test_vmulpd_mask_rd: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmulpd {rd-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <8 x double> @llvm.x86.avx512.mask.mul.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> zeroinitializer, i8 %mask, i32 1) ret <8 x double> %res } define <8 x double> @test_vmulpd_mask_ru(<8 x double> %a0, <8 x double> %a1, i8 %mask) { - ; CHECK-LABEL: test_vmulpd_mask_ru - ; CHECK: vmulpd {ru-sae}{{.*}}{%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xd9,0x59,0xc1] +; CHECK-LABEL: test_vmulpd_mask_ru: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmulpd {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <8 x double> @llvm.x86.avx512.mask.mul.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> zeroinitializer, i8 %mask, i32 2) ret <8 x double> %res } define <8 x double> @test_vmulpd_mask_rz(<8 x double> %a0, <8 x double> %a1, i8 %mask) { - ; CHECK-LABEL: test_vmulpd_mask_rz - ; CHECK: vmulpd {rz-sae}{{.*}}{%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xf9,0x59,0xc1] +; CHECK-LABEL: test_vmulpd_mask_rz: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmulpd {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <8 x double> @llvm.x86.avx512.mask.mul.pd.512(<8 x double> %a0, <8 x double> %a1, <8 x double> zeroinitializer, i8 %mask, i32 3) ret <8 x double> %res } define <16 x i32> @test_xor_epi32(<16 x i32> %a, <16 x i32> %b) { - ;CHECK-LABEL: test_xor_epi32 - ;CHECK: vpxord {{.*}}encoding: [0x62,0xf1,0x7d,0x48,0xef,0xc1] +; CHECK-LABEL: test_xor_epi32: +; CHECK: ## BB#0: +; CHECK-NEXT: vpxord %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.pxor.d.512(<16 x i32> %a,<16 x i32> %b, <16 x i32>zeroinitializer, i16 -1) ret < 16 x i32> %res } define <16 x i32> @test_mask_xor_epi32(<16 x i32> %a,<16 x i32> %b, <16 x i32> %passThru, i16 %mask) { - ;CHECK-LABEL: test_mask_xor_epi32 - ;CHECK: vpxord %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0xef,0xd1] +; CHECK-LABEL: test_mask_xor_epi32: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpxord %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.pxor.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask) ret < 16 x i32> %res } @@ -1544,15 +2416,21 @@ define <16 x i32> @test_mask_xor_epi32(<16 x i32> %a,<16 x i32> %b, <16 x i32> % declare <16 x i32> @llvm.x86.avx512.mask.pxor.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16) define <16 x i32> @test_or_epi32(<16 x i32> %a, <16 x i32> %b) { - ;CHECK-LABEL: test_or_epi32 - ;CHECK: vpord {{.*}}encoding: [0x62,0xf1,0x7d,0x48,0xeb,0xc1] +; CHECK-LABEL: test_or_epi32: +; CHECK: ## BB#0: +; CHECK-NEXT: vpord %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.por.d.512(<16 x i32> %a,<16 x i32> %b, <16 x i32>zeroinitializer, i16 -1) ret < 16 x i32> %res } define <16 x i32> @test_mask_or_epi32(<16 x i32> %a,<16 x i32> %b, <16 x i32> %passThru, i16 %mask) { - ;CHECK-LABEL: test_mask_or_epi32 - ;CHECK: vpord %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0xeb,0xd1] +; CHECK-LABEL: test_mask_or_epi32: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpord %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.por.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask) ret < 16 x i32> %res } @@ -1560,15 +2438,21 @@ define <16 x i32> @test_mask_or_epi32(<16 x i32> %a,<16 x i32> %b, <16 x i32> %p declare <16 x i32> @llvm.x86.avx512.mask.por.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16) define <16 x i32> @test_and_epi32(<16 x i32> %a, <16 x i32> %b) { - ;CHECK-LABEL: test_and_epi32 - ;CHECK: vpandd {{.*}}encoding: [0x62,0xf1,0x7d,0x48,0xdb,0xc1] +; CHECK-LABEL: test_and_epi32: +; CHECK: ## BB#0: +; CHECK-NEXT: vpandd %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.pand.d.512(<16 x i32> %a,<16 x i32> %b, <16 x i32>zeroinitializer, i16 -1) ret < 16 x i32> %res } define <16 x i32> @test_mask_and_epi32(<16 x i32> %a,<16 x i32> %b, <16 x i32> %passThru, i16 %mask) { - ;CHECK-LABEL: test_mask_and_epi32 - ;CHECK: vpandd %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0xdb,0xd1] +; CHECK-LABEL: test_mask_and_epi32: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpandd %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.pand.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask) ret < 16 x i32> %res } @@ -1576,15 +2460,22 @@ define <16 x i32> @test_mask_and_epi32(<16 x i32> %a,<16 x i32> %b, <16 x i32> % declare <16 x i32> @llvm.x86.avx512.mask.pand.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16) define <8 x i64> @test_xor_epi64(<8 x i64> %a, <8 x i64> %b) { - ;CHECK-LABEL: test_xor_epi64 - ;CHECK: vpxorq {{.*}}encoding: [0x62,0xf1,0xfd,0x48,0xef,0xc1] +; CHECK-LABEL: test_xor_epi64: +; CHECK: ## BB#0: +; CHECK-NEXT: vpxorq %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pxor.q.512(<8 x i64> %a,<8 x i64> %b, <8 x i64>zeroinitializer, i8 -1) ret < 8 x i64> %res } define <8 x i64> @test_mask_xor_epi64(<8 x i64> %a,<8 x i64> %b, <8 x i64> %passThru, i8 %mask) { - ;CHECK-LABEL: test_mask_xor_epi64 - ;CHECK: vpxorq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xef,0xd1] +; CHECK-LABEL: test_mask_xor_epi64: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpxorq %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pxor.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask) ret < 8 x i64> %res } @@ -1592,15 +2483,22 @@ define <8 x i64> @test_mask_xor_epi64(<8 x i64> %a,<8 x i64> %b, <8 x i64> %pass declare <8 x i64> @llvm.x86.avx512.mask.pxor.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8) define <8 x i64> @test_or_epi64(<8 x i64> %a, <8 x i64> %b) { - ;CHECK-LABEL: test_or_epi64 - ;CHECK: vporq {{.*}}encoding: [0x62,0xf1,0xfd,0x48,0xeb,0xc1] +; CHECK-LABEL: test_or_epi64: +; CHECK: ## BB#0: +; CHECK-NEXT: vporq %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.por.q.512(<8 x i64> %a,<8 x i64> %b, <8 x i64>zeroinitializer, i8 -1) ret < 8 x i64> %res } define <8 x i64> @test_mask_or_epi64(<8 x i64> %a,<8 x i64> %b, <8 x i64> %passThru, i8 %mask) { - ;CHECK-LABEL: test_mask_or_epi64 - ;CHECK: vporq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xeb,0xd1] +; CHECK-LABEL: test_mask_or_epi64: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vporq %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.por.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask) ret < 8 x i64> %res } @@ -1608,15 +2506,22 @@ define <8 x i64> @test_mask_or_epi64(<8 x i64> %a,<8 x i64> %b, <8 x i64> %passT declare <8 x i64> @llvm.x86.avx512.mask.por.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8) define <8 x i64> @test_and_epi64(<8 x i64> %a, <8 x i64> %b) { - ;CHECK-LABEL: test_and_epi64 - ;CHECK: vpandq {{.*}}encoding: [0x62,0xf1,0xfd,0x48,0xdb,0xc1] +; CHECK-LABEL: test_and_epi64: +; CHECK: ## BB#0: +; CHECK-NEXT: vpandq %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pand.q.512(<8 x i64> %a,<8 x i64> %b, <8 x i64>zeroinitializer, i8 -1) ret < 8 x i64> %res } define <8 x i64> @test_mask_and_epi64(<8 x i64> %a,<8 x i64> %b, <8 x i64> %passThru, i8 %mask) { - ;CHECK-LABEL: test_mask_and_epi64 - ;CHECK: vpandq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xdb,0xd1] +; CHECK-LABEL: test_mask_and_epi64: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpandq %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pand.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask) ret < 8 x i64> %res } @@ -1625,53 +2530,73 @@ declare <8 x i64> @llvm.x86.avx512.mask.pand.q.512(<8 x i64>, <8 x i64>, <8 x i6 define <16 x i32> @test_mask_add_epi32_rr(<16 x i32> %a, <16 x i32> %b) { - ;CHECK-LABEL: test_mask_add_epi32_rr - ;CHECK: vpaddd %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc1] +; CHECK-LABEL: test_mask_add_epi32_rr: +; CHECK: ## BB#0: +; CHECK-NEXT: vpaddd %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.padd.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1) ret < 16 x i32> %res } define <16 x i32> @test_mask_add_epi32_rrk(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask) { - ;CHECK-LABEL: test_mask_add_epi32_rrk - ;CHECK: vpaddd %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0xfe,0xd1] +; CHECK-LABEL: test_mask_add_epi32_rrk: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpaddd %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.padd.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask) ret < 16 x i32> %res } define <16 x i32> @test_mask_add_epi32_rrkz(<16 x i32> %a, <16 x i32> %b, i16 %mask) { - ;CHECK-LABEL: test_mask_add_epi32_rrkz - ;CHECK: vpaddd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0xfe,0xc1] +; CHECK-LABEL: test_mask_add_epi32_rrkz: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpaddd %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.padd.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask) ret < 16 x i32> %res } define <16 x i32> @test_mask_add_epi32_rm(<16 x i32> %a, <16 x i32>* %ptr_b) { - ;CHECK-LABEL: test_mask_add_epi32_rm - ;CHECK: vpaddd (%rdi), %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0x07] +; CHECK-LABEL: test_mask_add_epi32_rm: +; CHECK: ## BB#0: +; CHECK-NEXT: vpaddd (%rdi), %zmm0, %zmm0 +; CHECK-NEXT: retq %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <16 x i32> @llvm.x86.avx512.mask.padd.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1) ret < 16 x i32> %res } define <16 x i32> @test_mask_add_epi32_rmk(<16 x i32> %a, <16 x i32>* %ptr_b, <16 x i32> %passThru, i16 %mask) { - ;CHECK-LABEL: test_mask_add_epi32_rmk - ;CHECK: vpaddd (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0xfe,0x0f] +; CHECK-LABEL: test_mask_add_epi32_rmk: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpaddd (%rdi), %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <16 x i32> @llvm.x86.avx512.mask.padd.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask) ret < 16 x i32> %res } define <16 x i32> @test_mask_add_epi32_rmkz(<16 x i32> %a, <16 x i32>* %ptr_b, i16 %mask) { - ;CHECK-LABEL: test_mask_add_epi32_rmkz - ;CHECK: vpaddd (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0xfe,0x07] +; CHECK-LABEL: test_mask_add_epi32_rmkz: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpaddd (%rdi), %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <16 x i32> @llvm.x86.avx512.mask.padd.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask) ret < 16 x i32> %res } define <16 x i32> @test_mask_add_epi32_rmb(<16 x i32> %a, i32* %ptr_b) { - ;CHECK-LABEL: test_mask_add_epi32_rmb - ;CHECK: vpaddd (%rdi){1to16}, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x58,0xfe,0x07] +; CHECK-LABEL: test_mask_add_epi32_rmb: +; CHECK: ## BB#0: +; CHECK-NEXT: vpaddd (%rdi){1to16}, %zmm0, %zmm0 +; CHECK-NEXT: retq %q = load i32, i32* %ptr_b %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0 %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer @@ -1680,8 +2605,12 @@ define <16 x i32> @test_mask_add_epi32_rmb(<16 x i32> %a, i32* %ptr_b) { } define <16 x i32> @test_mask_add_epi32_rmbk(<16 x i32> %a, i32* %ptr_b, <16 x i32> %passThru, i16 %mask) { - ;CHECK-LABEL: test_mask_add_epi32_rmbk - ;CHECK: vpaddd (%rdi){1to16}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x59,0xfe,0x0f] +; CHECK-LABEL: test_mask_add_epi32_rmbk: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpaddd (%rdi){1to16}, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %q = load i32, i32* %ptr_b %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0 %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer @@ -1690,8 +2619,11 @@ define <16 x i32> @test_mask_add_epi32_rmbk(<16 x i32> %a, i32* %ptr_b, <16 x i3 } define <16 x i32> @test_mask_add_epi32_rmbkz(<16 x i32> %a, i32* %ptr_b, i16 %mask) { - ;CHECK-LABEL: test_mask_add_epi32_rmbkz - ;CHECK: vpaddd (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xd9,0xfe,0x07] +; CHECK-LABEL: test_mask_add_epi32_rmbkz: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpaddd (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %q = load i32, i32* %ptr_b %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0 %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer @@ -1702,53 +2634,73 @@ define <16 x i32> @test_mask_add_epi32_rmbkz(<16 x i32> %a, i32* %ptr_b, i16 %ma declare <16 x i32> @llvm.x86.avx512.mask.padd.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16) define <16 x i32> @test_mask_sub_epi32_rr(<16 x i32> %a, <16 x i32> %b) { - ;CHECK-LABEL: test_mask_sub_epi32_rr - ;CHECK: vpsubd %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfa,0xc1] +; CHECK-LABEL: test_mask_sub_epi32_rr: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsubd %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psub.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1) ret < 16 x i32> %res } define <16 x i32> @test_mask_sub_epi32_rrk(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask) { - ;CHECK-LABEL: test_mask_sub_epi32_rrk - ;CHECK: vpsubd %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0xfa,0xd1] +; CHECK-LABEL: test_mask_sub_epi32_rrk: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpsubd %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psub.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask) ret < 16 x i32> %res } define <16 x i32> @test_mask_sub_epi32_rrkz(<16 x i32> %a, <16 x i32> %b, i16 %mask) { - ;CHECK-LABEL: test_mask_sub_epi32_rrkz - ;CHECK: vpsubd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0xfa,0xc1] +; CHECK-LABEL: test_mask_sub_epi32_rrkz: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpsubd %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.psub.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask) ret < 16 x i32> %res } define <16 x i32> @test_mask_sub_epi32_rm(<16 x i32> %a, <16 x i32>* %ptr_b) { - ;CHECK-LABEL: test_mask_sub_epi32_rm - ;CHECK: vpsubd (%rdi), %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfa,0x07] +; CHECK-LABEL: test_mask_sub_epi32_rm: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsubd (%rdi), %zmm0, %zmm0 +; CHECK-NEXT: retq %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <16 x i32> @llvm.x86.avx512.mask.psub.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1) ret < 16 x i32> %res } define <16 x i32> @test_mask_sub_epi32_rmk(<16 x i32> %a, <16 x i32>* %ptr_b, <16 x i32> %passThru, i16 %mask) { - ;CHECK-LABEL: test_mask_sub_epi32_rmk - ;CHECK: vpsubd (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0xfa,0x0f] +; CHECK-LABEL: test_mask_sub_epi32_rmk: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpsubd (%rdi), %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <16 x i32> @llvm.x86.avx512.mask.psub.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask) ret < 16 x i32> %res } define <16 x i32> @test_mask_sub_epi32_rmkz(<16 x i32> %a, <16 x i32>* %ptr_b, i16 %mask) { - ;CHECK-LABEL: test_mask_sub_epi32_rmkz - ;CHECK: vpsubd (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0xfa,0x07] +; CHECK-LABEL: test_mask_sub_epi32_rmkz: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpsubd (%rdi), %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <16 x i32> @llvm.x86.avx512.mask.psub.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask) ret < 16 x i32> %res } define <16 x i32> @test_mask_sub_epi32_rmb(<16 x i32> %a, i32* %ptr_b) { - ;CHECK-LABEL: test_mask_sub_epi32_rmb - ;CHECK: vpsubd (%rdi){1to16}, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x58,0xfa,0x07] +; CHECK-LABEL: test_mask_sub_epi32_rmb: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsubd (%rdi){1to16}, %zmm0, %zmm0 +; CHECK-NEXT: retq %q = load i32, i32* %ptr_b %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0 %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer @@ -1757,8 +2709,12 @@ define <16 x i32> @test_mask_sub_epi32_rmb(<16 x i32> %a, i32* %ptr_b) { } define <16 x i32> @test_mask_sub_epi32_rmbk(<16 x i32> %a, i32* %ptr_b, <16 x i32> %passThru, i16 %mask) { - ;CHECK-LABEL: test_mask_sub_epi32_rmbk - ;CHECK: vpsubd (%rdi){1to16}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x59,0xfa,0x0f] +; CHECK-LABEL: test_mask_sub_epi32_rmbk: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpsubd (%rdi){1to16}, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %q = load i32, i32* %ptr_b %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0 %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer @@ -1767,8 +2723,11 @@ define <16 x i32> @test_mask_sub_epi32_rmbk(<16 x i32> %a, i32* %ptr_b, <16 x i3 } define <16 x i32> @test_mask_sub_epi32_rmbkz(<16 x i32> %a, i32* %ptr_b, i16 %mask) { - ;CHECK-LABEL: test_mask_sub_epi32_rmbkz - ;CHECK: vpsubd (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xd9,0xfa,0x07] +; CHECK-LABEL: test_mask_sub_epi32_rmbkz: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpsubd (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %q = load i32, i32* %ptr_b %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0 %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer @@ -1779,53 +2738,77 @@ define <16 x i32> @test_mask_sub_epi32_rmbkz(<16 x i32> %a, i32* %ptr_b, i16 %ma declare <16 x i32> @llvm.x86.avx512.mask.psub.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16) define <8 x i64> @test_mask_add_epi64_rr(<8 x i64> %a, <8 x i64> %b) { - ;CHECK-LABEL: test_mask_add_epi64_rr - ;CHECK: vpaddq %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc1] +; CHECK-LABEL: test_mask_add_epi64_rr: +; CHECK: ## BB#0: +; CHECK-NEXT: vpaddq %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.padd.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 -1) ret < 8 x i64> %res } define <8 x i64> @test_mask_add_epi64_rrk(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask) { - ;CHECK-LABEL: test_mask_add_epi64_rrk - ;CHECK: vpaddq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xd4,0xd1] +; CHECK-LABEL: test_mask_add_epi64_rrk: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpaddq %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.padd.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask) ret < 8 x i64> %res } define <8 x i64> @test_mask_add_epi64_rrkz(<8 x i64> %a, <8 x i64> %b, i8 %mask) { - ;CHECK-LABEL: test_mask_add_epi64_rrkz - ;CHECK: vpaddq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0xd4,0xc1] +; CHECK-LABEL: test_mask_add_epi64_rrkz: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpaddq %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.padd.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 %mask) ret < 8 x i64> %res } define <8 x i64> @test_mask_add_epi64_rm(<8 x i64> %a, <8 x i64>* %ptr_b) { - ;CHECK-LABEL: test_mask_add_epi64_rm - ;CHECK: vpaddq (%rdi), %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0x07] +; CHECK-LABEL: test_mask_add_epi64_rm: +; CHECK: ## BB#0: +; CHECK-NEXT: vpaddq (%rdi), %zmm0, %zmm0 +; CHECK-NEXT: retq %b = load <8 x i64>, <8 x i64>* %ptr_b %res = call <8 x i64> @llvm.x86.avx512.mask.padd.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 -1) ret < 8 x i64> %res } define <8 x i64> @test_mask_add_epi64_rmk(<8 x i64> %a, <8 x i64>* %ptr_b, <8 x i64> %passThru, i8 %mask) { - ;CHECK-LABEL: test_mask_add_epi64_rmk - ;CHECK: vpaddq (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xd4,0x0f] +; CHECK-LABEL: test_mask_add_epi64_rmk: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpaddq (%rdi), %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %b = load <8 x i64>, <8 x i64>* %ptr_b %res = call <8 x i64> @llvm.x86.avx512.mask.padd.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask) ret < 8 x i64> %res } define <8 x i64> @test_mask_add_epi64_rmkz(<8 x i64> %a, <8 x i64>* %ptr_b, i8 %mask) { - ;CHECK-LABEL: test_mask_add_epi64_rmkz - ;CHECK: vpaddq (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0xd4,0x07] +; CHECK-LABEL: test_mask_add_epi64_rmkz: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpaddq (%rdi), %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %b = load <8 x i64>, <8 x i64>* %ptr_b %res = call <8 x i64> @llvm.x86.avx512.mask.padd.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 %mask) ret < 8 x i64> %res } define <8 x i64> @test_mask_add_epi64_rmb(<8 x i64> %a, i64* %ptr_b) { - ;CHECK-LABEL: test_mask_add_epi64_rmb - ;CHECK: vpaddq (%rdi){1to8}, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x58,0xd4,0x07] +; CHECK-LABEL: test_mask_add_epi64_rmb: +; CHECK: ## BB#0: +; CHECK-NEXT: vpaddq (%rdi){1to8}, %zmm0, %zmm0 +; CHECK-NEXT: retq %q = load i64, i64* %ptr_b %vecinit.i = insertelement <8 x i64> undef, i64 %q, i32 0 %b = shufflevector <8 x i64> %vecinit.i, <8 x i64> undef, <8 x i32> zeroinitializer @@ -1834,8 +2817,13 @@ define <8 x i64> @test_mask_add_epi64_rmb(<8 x i64> %a, i64* %ptr_b) { } define <8 x i64> @test_mask_add_epi64_rmbk(<8 x i64> %a, i64* %ptr_b, <8 x i64> %passThru, i8 %mask) { - ;CHECK-LABEL: test_mask_add_epi64_rmbk - ;CHECK: vpaddq (%rdi){1to8}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x59,0xd4,0x0f] +; CHECK-LABEL: test_mask_add_epi64_rmbk: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpaddq (%rdi){1to8}, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %q = load i64, i64* %ptr_b %vecinit.i = insertelement <8 x i64> undef, i64 %q, i32 0 %b = shufflevector <8 x i64> %vecinit.i, <8 x i64> undef, <8 x i32> zeroinitializer @@ -1844,8 +2832,12 @@ define <8 x i64> @test_mask_add_epi64_rmbk(<8 x i64> %a, i64* %ptr_b, <8 x i64> } define <8 x i64> @test_mask_add_epi64_rmbkz(<8 x i64> %a, i64* %ptr_b, i8 %mask) { - ;CHECK-LABEL: test_mask_add_epi64_rmbkz - ;CHECK: vpaddq (%rdi){1to8}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xd9,0xd4,0x07] +; CHECK-LABEL: test_mask_add_epi64_rmbkz: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpaddq (%rdi){1to8}, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %q = load i64, i64* %ptr_b %vecinit.i = insertelement <8 x i64> undef, i64 %q, i32 0 %b = shufflevector <8 x i64> %vecinit.i, <8 x i64> undef, <8 x i32> zeroinitializer @@ -1856,53 +2848,77 @@ define <8 x i64> @test_mask_add_epi64_rmbkz(<8 x i64> %a, i64* %ptr_b, i8 %mask) declare <8 x i64> @llvm.x86.avx512.mask.padd.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8) define <8 x i64> @test_mask_sub_epi64_rr(<8 x i64> %a, <8 x i64> %b) { - ;CHECK-LABEL: test_mask_sub_epi64_rr - ;CHECK: vpsubq %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xfb,0xc1] +; CHECK-LABEL: test_mask_sub_epi64_rr: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsubq %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psub.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 -1) ret < 8 x i64> %res } define <8 x i64> @test_mask_sub_epi64_rrk(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask) { - ;CHECK-LABEL: test_mask_sub_epi64_rrk - ;CHECK: vpsubq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xfb,0xd1] +; CHECK-LABEL: test_mask_sub_epi64_rrk: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsubq %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psub.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask) ret < 8 x i64> %res } define <8 x i64> @test_mask_sub_epi64_rrkz(<8 x i64> %a, <8 x i64> %b, i8 %mask) { - ;CHECK-LABEL: test_mask_sub_epi64_rrkz - ;CHECK: vpsubq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0xfb,0xc1] +; CHECK-LABEL: test_mask_sub_epi64_rrkz: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsubq %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.psub.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 %mask) ret < 8 x i64> %res } define <8 x i64> @test_mask_sub_epi64_rm(<8 x i64> %a, <8 x i64>* %ptr_b) { - ;CHECK-LABEL: test_mask_sub_epi64_rm - ;CHECK: vpsubq (%rdi), %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xfb,0x07] +; CHECK-LABEL: test_mask_sub_epi64_rm: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsubq (%rdi), %zmm0, %zmm0 +; CHECK-NEXT: retq %b = load <8 x i64>, <8 x i64>* %ptr_b %res = call <8 x i64> @llvm.x86.avx512.mask.psub.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 -1) ret < 8 x i64> %res } define <8 x i64> @test_mask_sub_epi64_rmk(<8 x i64> %a, <8 x i64>* %ptr_b, <8 x i64> %passThru, i8 %mask) { - ;CHECK-LABEL: test_mask_sub_epi64_rmk - ;CHECK: vpsubq (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xfb,0x0f] +; CHECK-LABEL: test_mask_sub_epi64_rmk: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsubq (%rdi), %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %b = load <8 x i64>, <8 x i64>* %ptr_b %res = call <8 x i64> @llvm.x86.avx512.mask.psub.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passThru, i8 %mask) ret < 8 x i64> %res } define <8 x i64> @test_mask_sub_epi64_rmkz(<8 x i64> %a, <8 x i64>* %ptr_b, i8 %mask) { - ;CHECK-LABEL: test_mask_sub_epi64_rmkz - ;CHECK: vpsubq (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0xfb,0x07] +; CHECK-LABEL: test_mask_sub_epi64_rmkz: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsubq (%rdi), %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %b = load <8 x i64>, <8 x i64>* %ptr_b %res = call <8 x i64> @llvm.x86.avx512.mask.psub.q.512(<8 x i64> %a, <8 x i64> %b, <8 x i64> zeroinitializer, i8 %mask) ret < 8 x i64> %res } define <8 x i64> @test_mask_sub_epi64_rmb(<8 x i64> %a, i64* %ptr_b) { - ;CHECK-LABEL: test_mask_sub_epi64_rmb - ;CHECK: vpsubq (%rdi){1to8}, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x58,0xfb,0x07] +; CHECK-LABEL: test_mask_sub_epi64_rmb: +; CHECK: ## BB#0: +; CHECK-NEXT: vpsubq (%rdi){1to8}, %zmm0, %zmm0 +; CHECK-NEXT: retq %q = load i64, i64* %ptr_b %vecinit.i = insertelement <8 x i64> undef, i64 %q, i32 0 %b = shufflevector <8 x i64> %vecinit.i, <8 x i64> undef, <8 x i32> zeroinitializer @@ -1911,8 +2927,13 @@ define <8 x i64> @test_mask_sub_epi64_rmb(<8 x i64> %a, i64* %ptr_b) { } define <8 x i64> @test_mask_sub_epi64_rmbk(<8 x i64> %a, i64* %ptr_b, <8 x i64> %passThru, i8 %mask) { - ;CHECK-LABEL: test_mask_sub_epi64_rmbk - ;CHECK: vpsubq (%rdi){1to8}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x59,0xfb,0x0f] +; CHECK-LABEL: test_mask_sub_epi64_rmbk: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsubq (%rdi){1to8}, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %q = load i64, i64* %ptr_b %vecinit.i = insertelement <8 x i64> undef, i64 %q, i32 0 %b = shufflevector <8 x i64> %vecinit.i, <8 x i64> undef, <8 x i32> zeroinitializer @@ -1921,8 +2942,12 @@ define <8 x i64> @test_mask_sub_epi64_rmbk(<8 x i64> %a, i64* %ptr_b, <8 x i64> } define <8 x i64> @test_mask_sub_epi64_rmbkz(<8 x i64> %a, i64* %ptr_b, i8 %mask) { - ;CHECK-LABEL: test_mask_sub_epi64_rmbkz - ;CHECK: vpsubq (%rdi){1to8}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xd9,0xfb,0x07] +; CHECK-LABEL: test_mask_sub_epi64_rmbkz: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsubq (%rdi){1to8}, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %q = load i64, i64* %ptr_b %vecinit.i = insertelement <8 x i64> undef, i64 %q, i32 0 %b = shufflevector <8 x i64> %vecinit.i, <8 x i64> undef, <8 x i32> zeroinitializer @@ -1933,53 +2958,77 @@ define <8 x i64> @test_mask_sub_epi64_rmbkz(<8 x i64> %a, i64* %ptr_b, i8 %mask) declare <8 x i64> @llvm.x86.avx512.mask.psub.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8) define <8 x i64> @test_mask_mul_epi32_rr(<16 x i32> %a, <16 x i32> %b) { - ;CHECK-LABEL: test_mask_mul_epi32_rr - ;CHECK: vpmuldq %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf2,0xfd,0x48,0x28,0xc1] +; CHECK-LABEL: test_mask_mul_epi32_rr: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmuldq %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 -1) ret < 8 x i64> %res } define <8 x i64> @test_mask_mul_epi32_rrk(<16 x i32> %a, <16 x i32> %b, <8 x i64> %passThru, i8 %mask) { - ;CHECK-LABEL: test_mask_mul_epi32_rrk - ;CHECK: vpmuldq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x28,0xd1] +; CHECK-LABEL: test_mask_mul_epi32_rrk: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmuldq %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> %passThru, i8 %mask) ret < 8 x i64> %res } define <8 x i64> @test_mask_mul_epi32_rrkz(<16 x i32> %a, <16 x i32> %b, i8 %mask) { - ;CHECK-LABEL: test_mask_mul_epi32_rrkz - ;CHECK: vpmuldq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x28,0xc1] +; CHECK-LABEL: test_mask_mul_epi32_rrkz: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmuldq %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 %mask) ret < 8 x i64> %res } define <8 x i64> @test_mask_mul_epi32_rm(<16 x i32> %a, <16 x i32>* %ptr_b) { - ;CHECK-LABEL: test_mask_mul_epi32_rm - ;CHECK: vpmuldq (%rdi), %zmm0, %zmm0 ## encoding: [0x62,0xf2,0xfd,0x48,0x28,0x07] +; CHECK-LABEL: test_mask_mul_epi32_rm: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmuldq (%rdi), %zmm0, %zmm0 +; CHECK-NEXT: retq %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 -1) ret < 8 x i64> %res } define <8 x i64> @test_mask_mul_epi32_rmk(<16 x i32> %a, <16 x i32>* %ptr_b, <8 x i64> %passThru, i8 %mask) { - ;CHECK-LABEL: test_mask_mul_epi32_rmk - ;CHECK: vpmuldq (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x28,0x0f] +; CHECK-LABEL: test_mask_mul_epi32_rmk: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmuldq (%rdi), %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> %passThru, i8 %mask) ret < 8 x i64> %res } define <8 x i64> @test_mask_mul_epi32_rmkz(<16 x i32> %a, <16 x i32>* %ptr_b, i8 %mask) { - ;CHECK-LABEL: test_mask_mul_epi32_rmkz - ;CHECK: vpmuldq (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x28,0x07] +; CHECK-LABEL: test_mask_mul_epi32_rmkz: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmuldq (%rdi), %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 %mask) ret < 8 x i64> %res } define <8 x i64> @test_mask_mul_epi32_rmb(<16 x i32> %a, i64* %ptr_b) { - ;CHECK-LABEL: test_mask_mul_epi32_rmb - ;CHECK: vpmuldq (%rdi){1to8}, %zmm0, %zmm0 ## encoding: [0x62,0xf2,0xfd,0x58,0x28,0x07] +; CHECK-LABEL: test_mask_mul_epi32_rmb: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmuldq (%rdi){1to8}, %zmm0, %zmm0 +; CHECK-NEXT: retq %q = load i64, i64* %ptr_b %vecinit.i = insertelement <8 x i64> undef, i64 %q, i32 0 %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> undef, <8 x i32> zeroinitializer @@ -1989,8 +3038,13 @@ define <8 x i64> @test_mask_mul_epi32_rmb(<16 x i32> %a, i64* %ptr_b) { } define <8 x i64> @test_mask_mul_epi32_rmbk(<16 x i32> %a, i64* %ptr_b, <8 x i64> %passThru, i8 %mask) { - ;CHECK-LABEL: test_mask_mul_epi32_rmbk - ;CHECK: vpmuldq (%rdi){1to8}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x59,0x28,0x0f] +; CHECK-LABEL: test_mask_mul_epi32_rmbk: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmuldq (%rdi){1to8}, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %q = load i64, i64* %ptr_b %vecinit.i = insertelement <8 x i64> undef, i64 %q, i32 0 %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> undef, <8 x i32> zeroinitializer @@ -2000,8 +3054,12 @@ define <8 x i64> @test_mask_mul_epi32_rmbk(<16 x i32> %a, i64* %ptr_b, <8 x i64> } define <8 x i64> @test_mask_mul_epi32_rmbkz(<16 x i32> %a, i64* %ptr_b, i8 %mask) { - ;CHECK-LABEL: test_mask_mul_epi32_rmbkz - ;CHECK: vpmuldq (%rdi){1to8}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xd9,0x28,0x07] +; CHECK-LABEL: test_mask_mul_epi32_rmbkz: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmuldq (%rdi){1to8}, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %q = load i64, i64* %ptr_b %vecinit.i = insertelement <8 x i64> undef, i64 %q, i32 0 %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> undef, <8 x i32> zeroinitializer @@ -2013,53 +3071,77 @@ define <8 x i64> @test_mask_mul_epi32_rmbkz(<16 x i32> %a, i64* %ptr_b, i8 %mask declare <8 x i64> @llvm.x86.avx512.mask.pmul.dq.512(<16 x i32>, <16 x i32>, <8 x i64>, i8) define <8 x i64> @test_mask_mul_epu32_rr(<16 x i32> %a, <16 x i32> %b) { - ;CHECK-LABEL: test_mask_mul_epu32_rr - ;CHECK: vpmuludq %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xf4,0xc1] - %res = call <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 -1) +; CHECK-LABEL: test_mask_mul_epu32_rr: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmuludq %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 -1) ret < 8 x i64> %res } define <8 x i64> @test_mask_mul_epu32_rrk(<16 x i32> %a, <16 x i32> %b, <8 x i64> %passThru, i8 %mask) { - ;CHECK-LABEL: test_mask_mul_epu32_rrk - ;CHECK: vpmuludq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xf4,0xd1] +; CHECK-LABEL: test_mask_mul_epu32_rrk: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmuludq %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> %passThru, i8 %mask) ret < 8 x i64> %res } define <8 x i64> @test_mask_mul_epu32_rrkz(<16 x i32> %a, <16 x i32> %b, i8 %mask) { - ;CHECK-LABEL: test_mask_mul_epu32_rrkz - ;CHECK: vpmuludq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0xf4,0xc1] +; CHECK-LABEL: test_mask_mul_epu32_rrkz: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmuludq %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 %mask) ret < 8 x i64> %res } define <8 x i64> @test_mask_mul_epu32_rm(<16 x i32> %a, <16 x i32>* %ptr_b) { - ;CHECK-LABEL: test_mask_mul_epu32_rm - ;CHECK: vpmuludq (%rdi), %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xf4,0x07] +; CHECK-LABEL: test_mask_mul_epu32_rm: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmuludq (%rdi), %zmm0, %zmm0 +; CHECK-NEXT: retq %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 -1) ret < 8 x i64> %res } define <8 x i64> @test_mask_mul_epu32_rmk(<16 x i32> %a, <16 x i32>* %ptr_b, <8 x i64> %passThru, i8 %mask) { - ;CHECK-LABEL: test_mask_mul_epu32_rmk - ;CHECK: vpmuludq (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xf4,0x0f] +; CHECK-LABEL: test_mask_mul_epu32_rmk: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmuludq (%rdi), %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> %passThru, i8 %mask) ret < 8 x i64> %res } define <8 x i64> @test_mask_mul_epu32_rmkz(<16 x i32> %a, <16 x i32>* %ptr_b, i8 %mask) { - ;CHECK-LABEL: test_mask_mul_epu32_rmkz - ;CHECK: vpmuludq (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0xf4,0x07] +; CHECK-LABEL: test_mask_mul_epu32_rmkz: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmuludq (%rdi), %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32> %a, <16 x i32> %b, <8 x i64> zeroinitializer, i8 %mask) ret < 8 x i64> %res } define <8 x i64> @test_mask_mul_epu32_rmb(<16 x i32> %a, i64* %ptr_b) { - ;CHECK-LABEL: test_mask_mul_epu32_rmb - ;CHECK: vpmuludq (%rdi){1to8}, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x58,0xf4,0x07] +; CHECK-LABEL: test_mask_mul_epu32_rmb: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmuludq (%rdi){1to8}, %zmm0, %zmm0 +; CHECK-NEXT: retq %q = load i64, i64* %ptr_b %vecinit.i = insertelement <8 x i64> undef, i64 %q, i32 0 %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> undef, <8 x i32> zeroinitializer @@ -2069,8 +3151,13 @@ define <8 x i64> @test_mask_mul_epu32_rmb(<16 x i32> %a, i64* %ptr_b) { } define <8 x i64> @test_mask_mul_epu32_rmbk(<16 x i32> %a, i64* %ptr_b, <8 x i64> %passThru, i8 %mask) { - ;CHECK-LABEL: test_mask_mul_epu32_rmbk - ;CHECK: vpmuludq (%rdi){1to8}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x59,0xf4,0x0f] +; CHECK-LABEL: test_mask_mul_epu32_rmbk: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmuludq (%rdi){1to8}, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %q = load i64, i64* %ptr_b %vecinit.i = insertelement <8 x i64> undef, i64 %q, i32 0 %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> undef, <8 x i32> zeroinitializer @@ -2080,8 +3167,12 @@ define <8 x i64> @test_mask_mul_epu32_rmbk(<16 x i32> %a, i64* %ptr_b, <8 x i64> } define <8 x i64> @test_mask_mul_epu32_rmbkz(<16 x i32> %a, i64* %ptr_b, i8 %mask) { - ;CHECK-LABEL: test_mask_mul_epu32_rmbkz - ;CHECK: vpmuludq (%rdi){1to8}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xd9,0xf4,0x07] +; CHECK-LABEL: test_mask_mul_epu32_rmbkz: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmuludq (%rdi){1to8}, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %q = load i64, i64* %ptr_b %vecinit.i = insertelement <8 x i64> undef, i64 %q, i32 0 %b64 = shufflevector <8 x i64> %vecinit.i, <8 x i64> undef, <8 x i32> zeroinitializer @@ -2093,53 +3184,73 @@ define <8 x i64> @test_mask_mul_epu32_rmbkz(<16 x i32> %a, i64* %ptr_b, i8 %mask declare <8 x i64> @llvm.x86.avx512.mask.pmulu.dq.512(<16 x i32>, <16 x i32>, <8 x i64>, i8) define <16 x i32> @test_mask_mullo_epi32_rr_512(<16 x i32> %a, <16 x i32> %b) { - ;CHECK-LABEL: test_mask_mullo_epi32_rr_512 - ;CHECK: vpmulld %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf2,0x7d,0x48,0x40,0xc1] +; CHECK-LABEL: test_mask_mullo_epi32_rr_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmulld %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1) ret <16 x i32> %res } define <16 x i32> @test_mask_mullo_epi32_rrk_512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask) { - ;CHECK-LABEL: test_mask_mullo_epi32_rrk_512 - ;CHECK: vpmulld %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x40,0xd1] +; CHECK-LABEL: test_mask_mullo_epi32_rrk_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpmulld %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask) ret < 16 x i32> %res } define <16 x i32> @test_mask_mullo_epi32_rrkz_512(<16 x i32> %a, <16 x i32> %b, i16 %mask) { - ;CHECK-LABEL: test_mask_mullo_epi32_rrkz_512 - ;CHECK: vpmulld %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x40,0xc1] +; CHECK-LABEL: test_mask_mullo_epi32_rrkz_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpmulld %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask) ret < 16 x i32> %res } define <16 x i32> @test_mask_mullo_epi32_rm_512(<16 x i32> %a, <16 x i32>* %ptr_b) { - ;CHECK-LABEL: test_mask_mullo_epi32_rm_512 - ;CHECK: vpmulld (%rdi), %zmm0, %zmm0 ## encoding: [0x62,0xf2,0x7d,0x48,0x40,0x07] +; CHECK-LABEL: test_mask_mullo_epi32_rm_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmulld (%rdi), %zmm0, %zmm0 +; CHECK-NEXT: retq %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 -1) ret < 16 x i32> %res } define <16 x i32> @test_mask_mullo_epi32_rmk_512(<16 x i32> %a, <16 x i32>* %ptr_b, <16 x i32> %passThru, i16 %mask) { - ;CHECK-LABEL: test_mask_mullo_epi32_rmk_512 - ;CHECK: vpmulld (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x40,0x0f] +; CHECK-LABEL: test_mask_mullo_epi32_rmk_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpmulld (%rdi), %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> %passThru, i16 %mask) ret < 16 x i32> %res } define <16 x i32> @test_mask_mullo_epi32_rmkz_512(<16 x i32> %a, <16 x i32>* %ptr_b, i16 %mask) { - ;CHECK-LABEL: test_mask_mullo_epi32_rmkz_512 - ;CHECK: vpmulld (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x40,0x07] +; CHECK-LABEL: test_mask_mullo_epi32_rmkz_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpmulld (%rdi), %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32> %a, <16 x i32> %b, <16 x i32> zeroinitializer, i16 %mask) ret < 16 x i32> %res } define <16 x i32> @test_mask_mullo_epi32_rmb_512(<16 x i32> %a, i32* %ptr_b) { - ;CHECK-LABEL: test_mask_mullo_epi32_rmb_512 - ;CHECK: vpmulld (%rdi){1to16}, %zmm0, %zmm0 ## encoding: [0x62,0xf2,0x7d,0x58,0x40,0x07] +; CHECK-LABEL: test_mask_mullo_epi32_rmb_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmulld (%rdi){1to16}, %zmm0, %zmm0 +; CHECK-NEXT: retq %q = load i32, i32* %ptr_b %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0 %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer @@ -2148,8 +3259,12 @@ define <16 x i32> @test_mask_mullo_epi32_rmb_512(<16 x i32> %a, i32* %ptr_b) { } define <16 x i32> @test_mask_mullo_epi32_rmbk_512(<16 x i32> %a, i32* %ptr_b, <16 x i32> %passThru, i16 %mask) { - ;CHECK-LABEL: test_mask_mullo_epi32_rmbk_512 - ;CHECK: vpmulld (%rdi){1to16}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x59,0x40,0x0f] +; CHECK-LABEL: test_mask_mullo_epi32_rmbk_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpmulld (%rdi){1to16}, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmovaps %zmm1, %zmm0 +; CHECK-NEXT: retq %q = load i32, i32* %ptr_b %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0 %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer @@ -2158,8 +3273,11 @@ define <16 x i32> @test_mask_mullo_epi32_rmbk_512(<16 x i32> %a, i32* %ptr_b, <1 } define <16 x i32> @test_mask_mullo_epi32_rmbkz_512(<16 x i32> %a, i32* %ptr_b, i16 %mask) { - ;CHECK-LABEL: test_mask_mullo_epi32_rmbkz_512 - ;CHECK: vpmulld (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xd9,0x40,0x07] +; CHECK-LABEL: test_mask_mullo_epi32_rmbkz_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpmulld (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %q = load i32, i32* %ptr_b %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0 %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer @@ -2170,359 +3288,515 @@ define <16 x i32> @test_mask_mullo_epi32_rmbkz_512(<16 x i32> %a, i32* %ptr_b, i declare <16 x i32> @llvm.x86.avx512.mask.pmull.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16) define <16 x float> @test_mm512_maskz_add_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_maskz_add_round_ps_rn_sae - ;CHECK: vaddps {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_mm512_maskz_add_round_ps_rn_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddps {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 0) ret <16 x float> %res } define <16 x float> @test_mm512_maskz_add_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_maskz_add_round_ps_rd_sae - ;CHECK: vaddps {rd-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_maskz_add_round_ps_rd_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddps {rd-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 1) ret <16 x float> %res } define <16 x float> @test_mm512_maskz_add_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_maskz_add_round_ps_ru_sae - ;CHECK: vaddps {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_mm512_maskz_add_round_ps_ru_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddps {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 2) ret <16 x float> %res } define <16 x float> @test_mm512_maskz_add_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_maskz_add_round_ps_rz_sae - ;CHECK: vaddps {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_mm512_maskz_add_round_ps_rz_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddps {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 3) ret <16 x float> %res } define <16 x float> @test_mm512_maskz_add_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_maskz_add_round_ps_current - ;CHECK: vaddps %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_mm512_maskz_add_round_ps_current: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddps %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 4) ret <16 x float> %res } define <16 x float> @test_mm512_mask_add_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_add_round_ps_rn_sae - ;CHECK: vaddps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_add_round_ps_rn_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 0) ret <16 x float> %res } define <16 x float> @test_mm512_mask_add_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_add_round_ps_rd_sae - ;CHECK: vaddps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_add_round_ps_rd_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 1) ret <16 x float> %res } define <16 x float> @test_mm512_mask_add_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_add_round_ps_ru_sae - ;CHECK: vaddps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_add_round_ps_ru_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 2) ret <16 x float> %res } define <16 x float> @test_mm512_mask_add_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_add_round_ps_rz_sae - ;CHECK: vaddps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_add_round_ps_rz_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 3) ret <16 x float> %res } define <16 x float> @test_mm512_mask_add_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_add_round_ps_current - ;CHECK: vaddps %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_add_round_ps_current: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddps %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 4) ret <16 x float> %res } define <16 x float> @test_mm512_add_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_add_round_ps_rn_sae - ;CHECK: vaddps {rn-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_add_round_ps_rn_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vaddps {rn-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 0) ret <16 x float> %res } define <16 x float> @test_mm512_add_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_add_round_ps_rd_sae - ;CHECK: vaddps {rd-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_add_round_ps_rd_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vaddps {rd-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 1) ret <16 x float> %res } define <16 x float> @test_mm512_add_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_add_round_ps_ru_sae - ;CHECK: vaddps {ru-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_add_round_ps_ru_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vaddps {ru-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 2) ret <16 x float> %res } define <16 x float> @test_mm512_add_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_add_round_ps_rz_sae - ;CHECK: vaddps {rz-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_add_round_ps_rz_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vaddps {rz-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 3) ret <16 x float> %res } define <16 x float> @test_mm512_add_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_add_round_ps_current - ;CHECK: vaddps %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_add_round_ps_current: +; CHECK: ## BB#0: +; CHECK-NEXT: vaddps %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 4) ret <16 x float> %res } declare <16 x float> @llvm.x86.avx512.mask.add.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32) define <16 x float> @test_mm512_mask_sub_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_sub_round_ps_rn_sae - ;CHECK: vsubps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_sub_round_ps_rn_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vsubps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 0) ret <16 x float> %res } define <16 x float> @test_mm512_mask_sub_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_sub_round_ps_rd_sae - ;CHECK: vsubps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_sub_round_ps_rd_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vsubps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 1) ret <16 x float> %res } define <16 x float> @test_mm512_mask_sub_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_sub_round_ps_ru_sae - ;CHECK: vsubps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_sub_round_ps_ru_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vsubps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 2) ret <16 x float> %res } define <16 x float> @test_mm512_mask_sub_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_sub_round_ps_rz_sae - ;CHECK: vsubps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_sub_round_ps_rz_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vsubps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 3) ret <16 x float> %res } define <16 x float> @test_mm512_mask_sub_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_sub_round_ps_current - ;CHECK: vsubps %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_sub_round_ps_current: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vsubps %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 4) ret <16 x float> %res } define <16 x float> @test_mm512_sub_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_sub_round_ps_rn_sae - ;CHECK: vsubps {rn-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_sub_round_ps_rn_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vsubps {rn-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 0) ret <16 x float> %res } define <16 x float> @test_mm512_sub_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_sub_round_ps_rd_sae - ;CHECK: vsubps {rd-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_sub_round_ps_rd_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vsubps {rd-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 1) ret <16 x float> %res } define <16 x float> @test_mm512_sub_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_sub_round_ps_ru_sae - ;CHECK: vsubps {ru-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_sub_round_ps_ru_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vsubps {ru-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 2) ret <16 x float> %res } define <16 x float> @test_mm512_sub_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_sub_round_ps_rz_sae - ;CHECK: vsubps {rz-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_sub_round_ps_rz_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vsubps {rz-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 3) ret <16 x float> %res } define <16 x float> @test_mm512_sub_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_sub_round_ps_current - ;CHECK: vsubps %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_sub_round_ps_current: +; CHECK: ## BB#0: +; CHECK-NEXT: vsubps %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.sub.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 4) ret <16 x float> %res } define <16 x float> @test_mm512_maskz_div_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_maskz_div_round_ps_rn_sae - ;CHECK: vdivps {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_mm512_maskz_div_round_ps_rn_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vdivps {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 0) ret <16 x float> %res } define <16 x float> @test_mm512_maskz_div_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_maskz_div_round_ps_rd_sae - ;CHECK: vdivps {rd-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_maskz_div_round_ps_rd_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vdivps {rd-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 1) ret <16 x float> %res } define <16 x float> @test_mm512_maskz_div_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_maskz_div_round_ps_ru_sae - ;CHECK: vdivps {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_mm512_maskz_div_round_ps_ru_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vdivps {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 2) ret <16 x float> %res } define <16 x float> @test_mm512_maskz_div_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_maskz_div_round_ps_rz_sae - ;CHECK: vdivps {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_mm512_maskz_div_round_ps_rz_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vdivps {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 3) ret <16 x float> %res } define <16 x float> @test_mm512_maskz_div_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_maskz_div_round_ps_current - ;CHECK: vdivps %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_mm512_maskz_div_round_ps_current: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vdivps %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 4) ret <16 x float> %res } define <16 x float> @test_mm512_mask_div_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_div_round_ps_rn_sae - ;CHECK: vdivps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_div_round_ps_rn_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vdivps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 0) ret <16 x float> %res } define <16 x float> @test_mm512_mask_div_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_div_round_ps_rd_sae - ;CHECK: vdivps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_div_round_ps_rd_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vdivps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 1) ret <16 x float> %res } define <16 x float> @test_mm512_mask_div_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_div_round_ps_ru_sae - ;CHECK: vdivps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_div_round_ps_ru_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vdivps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 2) ret <16 x float> %res } define <16 x float> @test_mm512_mask_div_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_div_round_ps_rz_sae - ;CHECK: vdivps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_div_round_ps_rz_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vdivps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 3) ret <16 x float> %res } define <16 x float> @test_mm512_mask_div_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_div_round_ps_current - ;CHECK: vdivps %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_div_round_ps_current: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vdivps %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 4) ret <16 x float> %res } define <16 x float> @test_mm512_div_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_div_round_ps_rn_sae - ;CHECK: vdivps {rn-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_div_round_ps_rn_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vdivps {rn-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 0) ret <16 x float> %res } define <16 x float> @test_mm512_div_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_div_round_ps_rd_sae - ;CHECK: vdivps {rd-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_div_round_ps_rd_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vdivps {rd-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 1) ret <16 x float> %res } define <16 x float> @test_mm512_div_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_div_round_ps_ru_sae - ;CHECK: vdivps {ru-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_div_round_ps_ru_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vdivps {ru-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 2) ret <16 x float> %res } define <16 x float> @test_mm512_div_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_div_round_ps_rz_sae - ;CHECK: vdivps {rz-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_div_round_ps_rz_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vdivps {rz-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 3) ret <16 x float> %res } define <16 x float> @test_mm512_div_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_div_round_ps_current - ;CHECK: vdivps %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_div_round_ps_current: +; CHECK: ## BB#0: +; CHECK-NEXT: vdivps %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 4) ret <16 x float> %res } declare <16 x float> @llvm.x86.avx512.mask.div.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32) define <16 x float> @test_mm512_maskz_min_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_maskz_min_round_ps_sae - ;CHECK: vminps {sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_mm512_maskz_min_round_ps_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vminps {sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 8) ret <16 x float> %res } define <16 x float> @test_mm512_maskz_min_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_maskz_min_round_ps_current - ;CHECK: vminps %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_mm512_maskz_min_round_ps_current: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vminps %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 4) ret <16 x float> %res } define <16 x float> @test_mm512_mask_min_round_ps_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_min_round_ps_sae - ;CHECK: vminps {sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_min_round_ps_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vminps {sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 8) ret <16 x float> %res } define <16 x float> @test_mm512_mask_min_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_min_round_ps_current - ;CHECK: vminps %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_min_round_ps_current: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vminps %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 4) ret <16 x float> %res } define <16 x float> @test_mm512_min_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_min_round_ps_sae - ;CHECK: vminps {sae}, %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_min_round_ps_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vminps {sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 8) ret <16 x float> %res } define <16 x float> @test_mm512_min_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_min_round_ps_current - ;CHECK: vminps %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_min_round_ps_current: +; CHECK: ## BB#0: +; CHECK-NEXT: vminps %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 4) ret <16 x float> %res } declare <16 x float> @llvm.x86.avx512.mask.min.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32) define <16 x float> @test_mm512_maskz_max_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_maskz_max_round_ps_sae - ;CHECK: vmaxps {sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_mm512_maskz_max_round_ps_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmaxps {sae}, %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 8) ret <16 x float> %res } define <16 x float> @test_mm512_maskz_max_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_maskz_max_round_ps_current - ;CHECK: vmaxps %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-LABEL: test_mm512_maskz_max_round_ps_current: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmaxps %zmm1, %zmm0, %zmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 %mask, i32 4) ret <16 x float> %res } define <16 x float> @test_mm512_mask_max_round_ps_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_max_round_ps_sae - ;CHECK: vmaxps {sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_max_round_ps_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmaxps {sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 8) ret <16 x float> %res } define <16 x float> @test_mm512_mask_max_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) { - ;CHECK-LABEL: test_mm512_mask_max_round_ps_current - ;CHECK: vmaxps %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-LABEL: test_mm512_mask_max_round_ps_current: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmaxps %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask, i32 4) ret <16 x float> %res } define <16 x float> @test_mm512_max_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_max_round_ps_sae - ;CHECK: vmaxps {sae}, %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_max_round_ps_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vmaxps {sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 8) ret <16 x float> %res } define <16 x float> @test_mm512_max_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) { - ;CHECK-LABEL: test_mm512_max_round_ps_current - ;CHECK: vmaxps %zmm1, %zmm0, %zmm0 +; CHECK-LABEL: test_mm512_max_round_ps_current: +; CHECK: ## BB#0: +; CHECK-NEXT: vmaxps %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float> %a0, <16 x float> %a1, <16 x float>zeroinitializer, i16 -1, i32 4) ret <16 x float> %res } @@ -2531,50 +3805,81 @@ declare <16 x float> @llvm.x86.avx512.mask.max.ps.512(<16 x float>, <16 x float> declare <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone define <4 x float> @test_mask_add_ss_rn(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) { -; CHECK-LABEL: test_mask_add_ss_rn -; CHECK: vaddss {rn-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-LABEL: test_mask_add_ss_rn: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddss {rn-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 0) ret <4 x float> %res } define <4 x float> @test_mask_add_ss_rd(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) { -; CHECK-LABEL: test_mask_add_ss_rd -; CHECK: vaddss {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-LABEL: test_mask_add_ss_rd: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddss {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 1) ret <4 x float> %res } define <4 x float> @test_mask_add_ss_ru(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) { -; CHECK-LABEL: test_mask_add_ss_ru -; CHECK: vaddss {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-LABEL: test_mask_add_ss_ru: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddss {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 2) ret <4 x float> %res } define <4 x float> @test_mask_add_ss_rz(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) { -; CHECK-LABEL: test_mask_add_ss_rz -; CHECK: vaddss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-LABEL: test_mask_add_ss_rz: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 3) ret <4 x float> %res } define <4 x float> @test_mask_add_ss_current(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) { -; CHECK-LABEL: test_mask_add_ss_current -; CHECK: vaddss %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-LABEL: test_mask_add_ss_current: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddss %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 4) ret <4 x float> %res } define <4 x float> @test_maskz_add_ss_rn(<4 x float> %a0, <4 x float> %a1, i8 %mask) { -; CHECK-LABEL: test_maskz_add_ss_rn -; CHECK: vaddss {rn-sae}, %xmm1, %xmm0, %xmm0 {%k1} {z} +; CHECK-LABEL: test_maskz_add_ss_rn: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddss {rn-sae}, %xmm1, %xmm0, %xmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 0) ret <4 x float> %res } define <4 x float> @test_add_ss_rn(<4 x float> %a0, <4 x float> %a1) { -; CHECK-LABEL: test_add_ss_rn -; CHECK: vaddss {rn-sae}, %xmm1, %xmm0, %xmm0 +; CHECK-LABEL: test_add_ss_rn: +; CHECK: ## BB#0: +; CHECK-NEXT: vaddss {rn-sae}, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 0) ret <4 x float> %res } @@ -2582,50 +3887,81 @@ define <4 x float> @test_add_ss_rn(<4 x float> %a0, <4 x float> %a1) { declare <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone define <2 x double> @test_mask_add_sd_rn(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) { -; CHECK-LABEL: test_mask_add_sd_rn -; CHECK: vaddsd {rn-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-LABEL: test_mask_add_sd_rn: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddsd {rn-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 0) ret <2 x double> %res } define <2 x double> @test_mask_add_sd_rd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) { -; CHECK-LABEL: test_mask_add_sd_rd -; CHECK: vaddsd {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-LABEL: test_mask_add_sd_rd: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddsd {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 1) ret <2 x double> %res } define <2 x double> @test_mask_add_sd_ru(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) { -; CHECK-LABEL: test_mask_add_sd_ru -; CHECK: vaddsd {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-LABEL: test_mask_add_sd_ru: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddsd {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 2) ret <2 x double> %res } define <2 x double> @test_mask_add_sd_rz(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) { -; CHECK-LABEL: test_mask_add_sd_rz -; CHECK: vaddsd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-LABEL: test_mask_add_sd_rz: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddsd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 3) ret <2 x double> %res } define <2 x double> @test_mask_add_sd_current(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) { -; CHECK-LABEL: test_mask_add_sd_current -; CHECK: vaddsd %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-LABEL: test_mask_add_sd_current: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddsd %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 4) ret <2 x double> %res } define <2 x double> @test_maskz_add_sd_rn(<2 x double> %a0, <2 x double> %a1, i8 %mask) { -; CHECK-LABEL: test_maskz_add_sd_rn -; CHECK: vaddsd {rn-sae}, %xmm1, %xmm0, %xmm0 {%k1} {z} +; CHECK-LABEL: test_maskz_add_sd_rn: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vaddsd {rn-sae}, %xmm1, %xmm0, %xmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 0) ret <2 x double> %res } define <2 x double> @test_add_sd_rn(<2 x double> %a0, <2 x double> %a1) { -; CHECK-LABEL: test_add_sd_rn -; CHECK: vaddsd {rn-sae}, %xmm1, %xmm0, %xmm0 +; CHECK-LABEL: test_add_sd_rn: +; CHECK: ## BB#0: +; CHECK-NEXT: vaddsd {rn-sae}, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 0) ret <2 x double> %res } @@ -2633,86 +3969,130 @@ define <2 x double> @test_add_sd_rn(<2 x double> %a0, <2 x double> %a1) { declare <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone define <4 x float> @test_mask_max_ss_sae(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) { -; CHECK-LABEL: test_mask_max_ss_sae -; CHECK: vmaxss {sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-LABEL: test_mask_max_ss_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmaxss {sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 8) ret <4 x float> %res } define <4 x float> @test_maskz_max_ss_sae(<4 x float> %a0, <4 x float> %a1, i8 %mask) { -; CHECK-LABEL: test_maskz_max_ss_sae -; CHECK: vmaxss {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z} +; CHECK-LABEL: test_maskz_max_ss_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmaxss {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 8) ret <4 x float> %res } define <4 x float> @test_max_ss_sae(<4 x float> %a0, <4 x float> %a1) { -; CHECK-LABEL: test_max_ss_sae -; CHECK: vmaxss {sae}, %xmm1, %xmm0, %xmm0 +; CHECK-LABEL: test_max_ss_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vmaxss {sae}, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 8) ret <4 x float> %res } define <4 x float> @test_mask_max_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) { -; CHECK-LABEL: test_mask_max_ss -; CHECK: vmaxss %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-LABEL: test_mask_max_ss: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmaxss %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 4) ret <4 x float> %res } define <4 x float> @test_maskz_max_ss(<4 x float> %a0, <4 x float> %a1, i8 %mask) { -; CHECK-LABEL: test_maskz_max_ss -; CHECK: vmaxss %xmm1, %xmm0, %xmm0 {%k1} {z} +; CHECK-LABEL: test_maskz_max_ss: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmaxss %xmm1, %xmm0, %xmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 4) ret <4 x float> %res } define <4 x float> @test_max_ss(<4 x float> %a0, <4 x float> %a1) { -; CHECK-LABEL: test_max_ss -; CHECK: vmaxss %xmm1, %xmm0, %xmm0 +; CHECK-LABEL: test_max_ss: +; CHECK: ## BB#0: +; CHECK-NEXT: vmaxss %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 4) ret <4 x float> %res } declare <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone define <2 x double> @test_mask_max_sd_sae(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) { -; CHECK-LABEL: test_mask_max_sd_sae -; CHECK: vmaxsd {sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-LABEL: test_mask_max_sd_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmaxsd {sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 8) ret <2 x double> %res } define <2 x double> @test_maskz_max_sd_sae(<2 x double> %a0, <2 x double> %a1, i8 %mask) { -; CHECK-LABEL: test_maskz_max_sd_sae -; CHECK: vmaxsd {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z} +; CHECK-LABEL: test_maskz_max_sd_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmaxsd {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 8) ret <2 x double> %res } define <2 x double> @test_max_sd_sae(<2 x double> %a0, <2 x double> %a1) { -; CHECK-LABEL: test_max_sd_sae -; CHECK: vmaxsd {sae}, %xmm1, %xmm0, %xmm0 +; CHECK-LABEL: test_max_sd_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vmaxsd {sae}, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 8) ret <2 x double> %res } define <2 x double> @test_mask_max_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) { -; CHECK-LABEL: test_mask_max_sd -; CHECK: vmaxsd %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-LABEL: test_mask_max_sd: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmaxsd %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 4) ret <2 x double> %res } define <2 x double> @test_maskz_max_sd(<2 x double> %a0, <2 x double> %a1, i8 %mask) { -; CHECK-LABEL: test_maskz_max_sd -; CHECK: vmaxsd %xmm1, %xmm0, %xmm0 {%k1} {z} +; CHECK-LABEL: test_maskz_max_sd: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmaxsd %xmm1, %xmm0, %xmm0 {%k1} {z} +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 4) ret <2 x double> %res } define <2 x double> @test_max_sd(<2 x double> %a0, <2 x double> %a1) { -; CHECK-LABEL: test_max_sd -; CHECK: vmaxsd %xmm1, %xmm0, %xmm0 +; CHECK-LABEL: test_max_sd: +; CHECK: ## BB#0: +; CHECK-NEXT: vmaxsd %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 4) ret <2 x double> %res } @@ -2720,8 +4100,8 @@ define <2 x double> @test_max_sd(<2 x double> %a0, <2 x double> %a1) { define <2 x double> @test_x86_avx512_cvtsi2sd32(<2 x double> %a, i32 %b) { ; CHECK-LABEL: test_x86_avx512_cvtsi2sd32: ; CHECK: ## BB#0: -; CHECK-NEXT: vcvtsi2sdl %edi, {rz-sae}, %xmm0, %xmm0 -; CHECK-NEXT: retq +; CHECK-NEXT: vcvtsi2sdl %edi, {rz-sae}, %xmm0, %xmm0 +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.avx512.cvtsi2sd32(<2 x double> %a, i32 %b, i32 3) ; <<<2 x double>> [#uses=1] ret <2 x double> %res } @@ -2730,8 +4110,8 @@ declare <2 x double> @llvm.x86.avx512.cvtsi2sd32(<2 x double>, i32, i32) nounwin define <2 x double> @test_x86_avx512_cvtsi2sd64(<2 x double> %a, i64 %b) { ; CHECK-LABEL: test_x86_avx512_cvtsi2sd64: ; CHECK: ## BB#0: -; CHECK-NEXT: vcvtsi2sdq %rdi, {rz-sae}, %xmm0, %xmm0 -; CHECK-NEXT: retq +; CHECK-NEXT: vcvtsi2sdq %rdi, {rz-sae}, %xmm0, %xmm0 +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.avx512.cvtsi2sd64(<2 x double> %a, i64 %b, i32 3) ; <<<2 x double>> [#uses=1] ret <2 x double> %res } @@ -2740,8 +4120,8 @@ declare <2 x double> @llvm.x86.avx512.cvtsi2sd64(<2 x double>, i64, i32) nounwin define <4 x float> @test_x86_avx512_cvtsi2ss32(<4 x float> %a, i32 %b) { ; CHECK-LABEL: test_x86_avx512_cvtsi2ss32: ; CHECK: ## BB#0: -; CHECK-NEXT: vcvtsi2ssl %edi, {rz-sae}, %xmm0, %xmm0 -; CHECK-NEXT: retq +; CHECK-NEXT: vcvtsi2ssl %edi, {rz-sae}, %xmm0, %xmm0 +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.cvtsi2ss32(<4 x float> %a, i32 %b, i32 3) ; <<<4 x float>> [#uses=1] ret <4 x float> %res } @@ -2750,8 +4130,8 @@ declare <4 x float> @llvm.x86.avx512.cvtsi2ss32(<4 x float>, i32, i32) nounwind define <4 x float> @test_x86_avx512_cvtsi2ss64(<4 x float> %a, i64 %b) { ; CHECK-LABEL: test_x86_avx512_cvtsi2ss64: ; CHECK: ## BB#0: -; CHECK-NEXT: vcvtsi2ssq %rdi, {rz-sae}, %xmm0, %xmm0 -; CHECK-NEXT: retq +; CHECK-NEXT: vcvtsi2ssq %rdi, {rz-sae}, %xmm0, %xmm0 +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.avx512.cvtsi2ss64(<4 x float> %a, i64 %b, i32 3) ; <<<4 x float>> [#uses=1] ret <4 x float> %res } @@ -2760,8 +4140,8 @@ declare <4 x float> @llvm.x86.avx512.cvtsi2ss64(<4 x float>, i64, i32) nounwind define <4 x float> @test_x86_avx512__mm_cvt_roundu32_ss (<4 x float> %a, i32 %b) ; CHECK-LABEL: test_x86_avx512__mm_cvt_roundu32_ss: ; CHECK: ## BB#0: -; CHECK-NEXT: vcvtusi2ssl %edi, {rd-sae}, %xmm0, %xmm0 -; CHECK-NEXT: retq +; CHECK-NEXT: vcvtusi2ssl %edi, {rd-sae}, %xmm0, %xmm0 +; CHECK-NEXT: retq { %res = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> %a, i32 %b, i32 1) ; <<<4 x float>> [#uses=1] ret <4 x float> %res @@ -2770,9 +4150,9 @@ define <4 x float> @test_x86_avx512__mm_cvt_roundu32_ss (<4 x float> %a, i32 %b) define <4 x float> @test_x86_avx512__mm_cvt_roundu32_ss_mem(<4 x float> %a, i32* %ptr) ; CHECK-LABEL: test_x86_avx512__mm_cvt_roundu32_ss_mem: ; CHECK: ## BB#0: -; CHECK-NEXT: movl (%rdi), %eax -; CHECK-NEXT: vcvtusi2ssl %eax, {rd-sae}, %xmm0, %xmm0 -; CHECK-NEXT: retq +; CHECK-NEXT: movl (%rdi), %eax +; CHECK-NEXT: vcvtusi2ssl %eax, {rd-sae}, %xmm0, %xmm0 +; CHECK-NEXT: retq { %b = load i32, i32* %ptr %res = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> %a, i32 %b, i32 1) ; <<<4 x float>> [#uses=1] @@ -2782,8 +4162,8 @@ define <4 x float> @test_x86_avx512__mm_cvt_roundu32_ss_mem(<4 x float> %a, i32* define <4 x float> @test_x86_avx512__mm_cvtu32_ss(<4 x float> %a, i32 %b) ; CHECK-LABEL: test_x86_avx512__mm_cvtu32_ss: ; CHECK: ## BB#0: -; CHECK-NEXT: vcvtusi2ssl %edi, %xmm0, %xmm0 -; CHECK-NEXT: retq +; CHECK-NEXT: vcvtusi2ssl %edi, %xmm0, %xmm0 +; CHECK-NEXT: retq { %res = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> %a, i32 %b, i32 4) ; <<<4 x float>> [#uses=1] ret <4 x float> %res @@ -2793,7 +4173,7 @@ define <4 x float> @test_x86_avx512__mm_cvtu32_ss_mem(<4 x float> %a, i32* %ptr) ; CHECK-LABEL: test_x86_avx512__mm_cvtu32_ss_mem: ; CHECK: ## BB#0: ; CHECK-NEXT: vcvtusi2ssl (%rdi), %xmm0, %xmm0 -; CHECK-NEXT: retq +; CHECK-NEXT: retq { %b = load i32, i32* %ptr %res = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> %a, i32 %b, i32 4) ; <<<4 x float>> [#uses=1] @@ -2804,8 +4184,8 @@ declare <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float>, i32, i32) nounwind r define <4 x float> @_mm_cvt_roundu64_ss (<4 x float> %a, i64 %b) ; CHECK-LABEL: _mm_cvt_roundu64_ss: ; CHECK: ## BB#0: -; CHECK-NEXT: vcvtusi2ssq %rdi, {rd-sae}, %xmm0, %xmm0 -; CHECK-NEXT: retq +; CHECK-NEXT: vcvtusi2ssq %rdi, {rd-sae}, %xmm0, %xmm0 +; CHECK-NEXT: retq { %res = call <4 x float> @llvm.x86.avx512.cvtusi642ss(<4 x float> %a, i64 %b, i32 1) ; <<<4 x float>> [#uses=1] ret <4 x float> %res @@ -2814,8 +4194,8 @@ define <4 x float> @_mm_cvt_roundu64_ss (<4 x float> %a, i64 %b) define <4 x float> @_mm_cvtu64_ss(<4 x float> %a, i64 %b) ; CHECK-LABEL: _mm_cvtu64_ss: ; CHECK: ## BB#0: -; CHECK-NEXT: vcvtusi2ssq %rdi, %xmm0, %xmm0 -; CHECK-NEXT: retq +; CHECK-NEXT: vcvtusi2ssq %rdi, %xmm0, %xmm0 +; CHECK-NEXT: retq { %res = call <4 x float> @llvm.x86.avx512.cvtusi642ss(<4 x float> %a, i64 %b, i32 4) ; <<<4 x float>> [#uses=1] ret <4 x float> %res @@ -2825,8 +4205,8 @@ declare <4 x float> @llvm.x86.avx512.cvtusi642ss(<4 x float>, i64, i32) nounwind define <2 x double> @test_x86_avx512_mm_cvtu32_sd(<2 x double> %a, i32 %b) ; CHECK-LABEL: test_x86_avx512_mm_cvtu32_sd: ; CHECK: ## BB#0: -; CHECK-NEXT: vcvtusi2sdl %edi, %xmm0, %xmm0 -; CHECK-NEXT: retq +; CHECK-NEXT: vcvtusi2sdl %edi, %xmm0, %xmm0 +; CHECK-NEXT: retq { %res = call <2 x double> @llvm.x86.avx512.cvtusi2sd(<2 x double> %a, i32 %b) ; <<<2 x double>> [#uses=1] ret <2 x double> %res @@ -2836,8 +4216,8 @@ declare <2 x double> @llvm.x86.avx512.cvtusi2sd(<2 x double>, i32) nounwind read define <2 x double> @test_x86_avx512_mm_cvtu64_sd(<2 x double> %a, i64 %b) ; CHECK-LABEL: test_x86_avx512_mm_cvtu64_sd: ; CHECK: ## BB#0: -; CHECK-NEXT: vcvtusi2sdq %rdi, {rd-sae}, %xmm0, %xmm0 -; CHECK-NEXT: retq +; CHECK-NEXT: vcvtusi2sdq %rdi, {rd-sae}, %xmm0, %xmm0 +; CHECK-NEXT: retq { %res = call <2 x double> @llvm.x86.avx512.cvtusi642sd(<2 x double> %a, i64 %b, i32 1) ; <<<2 x double>> [#uses=1] ret <2 x double> %res @@ -2846,8 +4226,8 @@ define <2 x double> @test_x86_avx512_mm_cvtu64_sd(<2 x double> %a, i64 %b) define <2 x double> @test_x86_avx512__mm_cvt_roundu64_sd(<2 x double> %a, i64 %b) ; CHECK-LABEL: test_x86_avx512__mm_cvt_roundu64_sd: ; CHECK: ## BB#0: -; CHECK-NEXT: vcvtusi2sdq %rdi, %xmm0, %xmm0 -; CHECK-NEXT: retq +; CHECK-NEXT: vcvtusi2sdq %rdi, %xmm0, %xmm0 +; CHECK-NEXT: retq { %res = call <2 x double> @llvm.x86.avx512.cvtusi642sd(<2 x double> %a, i64 %b, i32 4) ; <<<2 x double>> [#uses=1] ret <2 x double> %res @@ -2855,7 +4235,10 @@ define <2 x double> @test_x86_avx512__mm_cvt_roundu64_sd(<2 x double> %a, i64 %b declare <2 x double> @llvm.x86.avx512.cvtusi642sd(<2 x double>, i64, i32) nounwind readnone define <8 x i64> @test_vpmaxq(<8 x i64> %a0, <8 x i64> %a1) { - ; CHECK: vpmaxsq {{.*}}encoding: [0x62,0xf2,0xfd,0x48,0x3d,0xc1] +; CHECK-LABEL: test_vpmaxq: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pmaxs.q.512(<8 x i64> %a0, <8 x i64> %a1, <8 x i64>zeroinitializer, i8 -1) ret <8 x i64> %res @@ -2863,7 +4246,10 @@ define <8 x i64> @test_vpmaxq(<8 x i64> %a0, <8 x i64> %a1) { declare <8 x i64> @llvm.x86.avx512.mask.pmaxs.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8) define <16 x i32> @test_vpminud(<16 x i32> %a0, <16 x i32> %a1) { - ; CHECK: vpminud {{.*}}encoding: [0x62,0xf2,0x7d,0x48,0x3b,0xc1] +; CHECK-LABEL: test_vpminud: +; CHECK: ## BB#0: +; CHECK-NEXT: vpminud %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.pminu.d.512(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>zeroinitializer, i16 -1) ret <16 x i32> %res @@ -2871,29 +4257,39 @@ define <16 x i32> @test_vpminud(<16 x i32> %a0, <16 x i32> %a1) { declare <16 x i32> @llvm.x86.avx512.mask.pminu.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16) define <16 x i32> @test_vpmaxsd(<16 x i32> %a0, <16 x i32> %a1) { - ; CHECK: vpmaxsd {{.*}}encoding: [0x62,0xf2,0x7d,0x48,0x3d,0xc1] +; CHECK-LABEL: test_vpmaxsd: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmaxsd %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.pmaxs.d.512(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>zeroinitializer, i16 -1) ret <16 x i32> %res } declare <16 x i32> @llvm.x86.avx512.mask.pmaxs.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16) -; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_d_512 -; CHECK-NOT: call -; CHECK: vpmaxsd %zmm -; CHECK: {%k1} define <16 x i32>@test_int_x86_avx512_mask_pmaxs_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmaxs_d_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpmaxsd %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vpmaxsd %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: vpaddd %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.pmaxs.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) %res1 = call <16 x i32> @llvm.x86.avx512.mask.pmaxs.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1) %res2 = add <16 x i32> %res, %res1 ret <16 x i32> %res2 } -; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_q_512 -; CHECK-NOT: call -; CHECK: vpmaxsq %zmm -; CHECK: {%k1} define <8 x i64>@test_int_x86_avx512_mask_pmaxs_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmaxs_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmaxsq %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pmaxs.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) %res1 = call <8 x i64> @llvm.x86.avx512.mask.pmaxs.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1) %res2 = add <8 x i64> %res, %res1 @@ -2902,11 +4298,14 @@ define <8 x i64>@test_int_x86_avx512_mask_pmaxs_q_512(<8 x i64> %x0, <8 x i64> % declare <16 x i32> @llvm.x86.avx512.mask.pmaxu.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16) -; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_d_512 -; CHECK-NOT: call -; CHECK: vpmaxud %zmm -; CHECK: {%k1} define <16 x i32>@test_int_x86_avx512_mask_pmaxu_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmaxu_d_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpmaxud %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vpmaxud %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: vpaddd %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.pmaxu.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) %res1 = call <16 x i32> @llvm.x86.avx512.mask.pmaxu.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1) %res2 = add <16 x i32> %res, %res1 @@ -2915,11 +4314,15 @@ define <16 x i32>@test_int_x86_avx512_mask_pmaxu_d_512(<16 x i32> %x0, <16 x i32 declare <8 x i64> @llvm.x86.avx512.mask.pmaxu.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8) -; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_q_512 -; CHECK-NOT: call -; CHECK: vpmaxuq %zmm -; CHECK: {%k1} define <8 x i64>@test_int_x86_avx512_mask_pmaxu_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmaxu_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmaxuq %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vpmaxuq %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pmaxu.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) %res1 = call <8 x i64> @llvm.x86.avx512.mask.pmaxu.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1) %res2 = add <8 x i64> %res, %res1 @@ -2928,11 +4331,14 @@ define <8 x i64>@test_int_x86_avx512_mask_pmaxu_q_512(<8 x i64> %x0, <8 x i64> % declare <16 x i32> @llvm.x86.avx512.mask.pmins.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16) -; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_d_512 -; CHECK-NOT: call -; CHECK: vpminsd %zmm -; CHECK: {%k1} define <16 x i32>@test_int_x86_avx512_mask_pmins_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmins_d_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpminsd %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vpminsd %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: vpaddd %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.pmins.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) %res1 = call <16 x i32> @llvm.x86.avx512.mask.pmins.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1) %res2 = add <16 x i32> %res, %res1 @@ -2941,22 +4347,29 @@ define <16 x i32>@test_int_x86_avx512_mask_pmins_d_512(<16 x i32> %x0, <16 x i32 declare <8 x i64> @llvm.x86.avx512.mask.pmins.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8) -; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_q_512 -; CHECK-NOT: call -; CHECK: vpminsq %zmm -; CHECK: {%k1} define <8 x i64>@test_int_x86_avx512_mask_pmins_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmins_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpminsq %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vpminsq %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pmins.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) %res1 = call <8 x i64> @llvm.x86.avx512.mask.pmins.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1) %res2 = add <8 x i64> %res, %res1 ret <8 x i64> %res2 } -; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_d_512 -; CHECK-NOT: call -; CHECK: vpminud %zmm -; CHECK: {%k1} define <16 x i32>@test_int_x86_avx512_mask_pminu_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_pminu_d_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpminud %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vpminud %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: vpaddd %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.pminu.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) %res1 = call <16 x i32> @llvm.x86.avx512.mask.pminu.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1) %res2 = add <16 x i32> %res, %res1 @@ -2965,11 +4378,15 @@ define <16 x i32>@test_int_x86_avx512_mask_pminu_d_512(<16 x i32> %x0, <16 x i32 declare <8 x i64> @llvm.x86.avx512.mask.pminu.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8) -; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_q_512 -; CHECK-NOT: call -; CHECK: vpminuq %zmm -; CHECK: {%k1} define <8 x i64>@test_int_x86_avx512_mask_pminu_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_pminu_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpminuq %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vpminuq %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.pminu.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) %res1 = call <8 x i64> @llvm.x86.avx512.mask.pminu.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1) %res2 = add <8 x i64> %res, %res1 @@ -2978,24 +4395,34 @@ define <8 x i64>@test_int_x86_avx512_mask_pminu_q_512(<8 x i64> %x0, <8 x i64> % declare <16 x i32> @llvm.x86.avx512.mask.vpermi2var.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16) -; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_d_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpermi2d {{.*}}{%k1} -define <16 x i32>@test_int_x86_avx512_mask_vpermi2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) { +define <16 x i32>@test_int_x86_avx512_mask_vpermi2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpermi2var_d_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vmovaps %zmm1, %zmm3 +; CHECK-NEXT: vpermi2d (%rdi), %zmm0, %zmm3 {%k1} +; CHECK-NEXT: vpermi2d %zmm2, %zmm0, %zmm1 +; CHECK-NEXT: vpaddd %zmm1, %zmm3, %zmm0 +; CHECK-NEXT: retq + %x2 = load <16 x i32>, <16 x i32>* %x2p %res = call <16 x i32> @llvm.x86.avx512.mask.vpermi2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) - %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpermi2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1) + %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpermi2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16 -1) %res2 = add <16 x i32> %res, %res1 ret <16 x i32> %res2 } declare <8 x double> @llvm.x86.avx512.mask.vpermi2var.pd.512(<8 x double>, <8 x i64>, <8 x double>, i8) -; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_pd_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpermi2pd {{.*}}{%k1} define <8 x double>@test_int_x86_avx512_mask_vpermi2var_pd_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpermi2var_pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovaps %zmm1, %zmm3 +; CHECK-NEXT: vpermi2pd %zmm2, %zmm0, %zmm3 {%k1} +; CHECK-NEXT: vpermi2pd %zmm2, %zmm0, %zmm1 +; CHECK-NEXT: vaddpd %zmm1, %zmm3, %zmm0 +; CHECK-NEXT: retq %res = call <8 x double> @llvm.x86.avx512.mask.vpermi2var.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3) %res1 = call <8 x double> @llvm.x86.avx512.mask.vpermi2var.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 -1) %res2 = fadd <8 x double> %res, %res1 @@ -3004,11 +4431,15 @@ define <8 x double>@test_int_x86_avx512_mask_vpermi2var_pd_512(<8 x double> %x0, declare <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float>, <16 x i32>, <16 x float>, i16) -; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_ps_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpermi2ps {{.*}}{%k1} define <16 x float>@test_int_x86_avx512_mask_vpermi2var_ps_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpermi2var_ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovaps %zmm1, %zmm3 +; CHECK-NEXT: vpermi2ps %zmm2, %zmm0, %zmm3 {%k1} +; CHECK-NEXT: vpermi2ps %zmm2, %zmm0, %zmm1 +; CHECK-NEXT: vaddps %zmm1, %zmm3, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3) %res1 = call <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 -1) %res2 = fadd <16 x float> %res, %res1 @@ -3017,11 +4448,16 @@ define <16 x float>@test_int_x86_avx512_mask_vpermi2var_ps_512(<16 x float> %x0, declare <8 x i64> @llvm.x86.avx512.mask.vpermi2var.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8) -; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_q_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpermi2q {{.*}}{%k1} define <8 x i64>@test_int_x86_avx512_mask_vpermi2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpermi2var_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovaps %zmm1, %zmm3 +; CHECK-NEXT: vpermi2q %zmm2, %zmm0, %zmm3 {%k1} +; CHECK-NEXT: vpermi2q %zmm2, %zmm0, %zmm1 +; CHECK-NEXT: vpaddq %zmm1, %zmm3, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.mask.vpermi2var.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) %res1 = call <8 x i64> @llvm.x86.avx512.mask.vpermi2var.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1) %res2 = add <8 x i64> %res, %res1 @@ -3030,37 +4466,54 @@ define <8 x i64>@test_int_x86_avx512_mask_vpermi2var_q_512(<8 x i64> %x0, <8 x i declare <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16) -; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_d_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpermt2d {{.*}}{%k1} {z} -define <16 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) { +define <16 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_maskz_vpermt2var_d_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vmovaps %zmm1, %zmm2 +; CHECK-NEXT: vpermt2d (%rdi), %zmm0, %zmm2 {%k1} {z} +; CHECK-NEXT: vpermt2d %zmm1, %zmm0, %zmm1 +; CHECK-NEXT: vpaddd %zmm1, %zmm2, %zmm0 +; CHECK-NEXT: retq + %x2 = load <16 x i32>, <16 x i32>* %x2p %res = call <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) - %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1) + %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x1, i16 -1) %res2 = add <16 x i32> %res, %res1 ret <16 x i32> %res2 } declare <8 x double> @llvm.x86.avx512.maskz.vpermt2var.pd.512(<8 x i64>, <8 x double>, <8 x double>, i8) -; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_pd_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpermt2pd {{.*}}{%k1} {z} -define <8 x double>@test_int_x86_avx512_maskz_vpermt2var_pd_512(<8 x i64> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3) { +define <8 x double>@test_int_x86_avx512_maskz_vpermt2var_pd_512(<8 x i64> %x0, <8 x double> %x1, double* %x2ptr, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_maskz_vpermt2var_pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovaps %zmm1, %zmm2 +; CHECK-NEXT: vpermt2pd (%rdi){1to8}, %zmm0, %zmm2 {%k1} {z} +; CHECK-NEXT: vpermt2pd %zmm1, %zmm0, %zmm1 +; CHECK-NEXT: vaddpd %zmm1, %zmm2, %zmm0 +; CHECK-NEXT: retq + %x2s = load double, double* %x2ptr + %x2ins = insertelement <8 x double> undef, double %x2s, i32 0 + %x2 = shufflevector <8 x double> %x2ins, <8 x double> undef, <8 x i32> zeroinitializer %res = call <8 x double> @llvm.x86.avx512.maskz.vpermt2var.pd.512(<8 x i64> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3) - %res1 = call <8 x double> @llvm.x86.avx512.maskz.vpermt2var.pd.512(<8 x i64> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1) + %res1 = call <8 x double> @llvm.x86.avx512.maskz.vpermt2var.pd.512(<8 x i64> %x0, <8 x double> %x1, <8 x double> %x1, i8 -1) %res2 = fadd <8 x double> %res, %res1 ret <8 x double> %res2 } declare <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32>, <16 x float>, <16 x float>, i16) -; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_ps_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpermt2ps {{.*}}{%k1} {z} define <16 x float>@test_int_x86_avx512_maskz_vpermt2var_ps_512(<16 x i32> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_maskz_vpermt2var_ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovaps %zmm1, %zmm3 +; CHECK-NEXT: vpermt2ps %zmm2, %zmm0, %zmm3 {%k1} {z} +; CHECK-NEXT: vpermt2ps %zmm2, %zmm0, %zmm1 +; CHECK-NEXT: vaddps %zmm1, %zmm3, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3) %res1 = call <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1) %res2 = fadd <16 x float> %res, %res1 @@ -3070,11 +4523,16 @@ define <16 x float>@test_int_x86_avx512_maskz_vpermt2var_ps_512(<16 x i32> %x0, declare <8 x i64> @llvm.x86.avx512.maskz.vpermt2var.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8) -; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_q_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpermt2q {{.*}}{%k1} {z} define <8 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_maskz_vpermt2var_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovaps %zmm1, %zmm3 +; CHECK-NEXT: vpermt2q %zmm2, %zmm0, %zmm3 {%k1} {z} +; CHECK-NEXT: vpermt2q %zmm2, %zmm0, %zmm1 +; CHECK-NEXT: vpaddq %zmm1, %zmm3, %zmm0 +; CHECK-NEXT: retq %res = call <8 x i64> @llvm.x86.avx512.maskz.vpermt2var.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) %res1 = call <8 x i64> @llvm.x86.avx512.maskz.vpermt2var.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1) %res2 = add <8 x i64> %res, %res1 @@ -3083,12 +4541,15 @@ define <8 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_512(<8 x i64> %x0, <8 x declare <16 x i32> @llvm.x86.avx512.mask.vpermt2var.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16) -; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_d_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpermt2d {{.*}}{%k1} -; CHECK-NOT: {z} define <16 x i32>@test_int_x86_avx512_mask_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpermt2var_d_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovaps %zmm1, %zmm3 +; CHECK-NEXT: vpermt2d %zmm2, %zmm0, %zmm3 {%k1} +; CHECK-NEXT: vpermt2d %zmm2, %zmm0, %zmm1 +; CHECK-NEXT: vpaddd %zmm1, %zmm3, %zmm0 +; CHECK-NEXT: retq %res = call <16 x i32> @llvm.x86.avx512.mask.vpermt2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpermt2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1) %res2 = add <16 x i32> %res, %res1 @@ -3096,11 +4557,15 @@ define <16 x i32>@test_int_x86_avx512_mask_vpermt2var_d_512(<16 x i32> %x0, <16 } declare <8 x double> @llvm.x86.avx512.mask.scalef.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32) -; CHECK-LABEL: @test_int_x86_avx512_mask_scalef_pd_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vscalefpd{{.*}}{%k1} define <8 x double>@test_int_x86_avx512_mask_scalef_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_scalef_pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vscalefpd {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vscalefpd {rn-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: vaddpd %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <8 x double> @llvm.x86.avx512.mask.scalef.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3, i32 3) %res1 = call <8 x double> @llvm.x86.avx512.mask.scalef.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1, i32 0) %res2 = fadd <8 x double> %res, %res1 @@ -3108,13 +4573,1849 @@ define <8 x double>@test_int_x86_avx512_mask_scalef_pd_512(<8 x double> %x0, <8 } declare <16 x float> @llvm.x86.avx512.mask.scalef.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32) -; CHECK-LABEL: @test_int_x86_avx512_mask_scalef_ps_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vscalefps{{.*}}{%k1} define <16 x float>@test_int_x86_avx512_mask_scalef_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_scalef_ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vscalefps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vscalefps {rn-sae}, %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: vaddps %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq %res = call <16 x float> @llvm.x86.avx512.mask.scalef.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3, i32 2) %res1 = call <16 x float> @llvm.x86.avx512.mask.scalef.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1, i32 0) %res2 = fadd <16 x float> %res, %res1 ret <16 x float> %res2 } + +declare <8 x double> @llvm.x86.avx512.mask.unpckh.pd.512(<8 x double>, <8 x double>, <8 x double>, i8) + +define <8 x double>@test_int_x86_avx512_mask_unpckh_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_unpckh_pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vunpckhpd {{.*#+}} zmm2 = zmm2[1],k1[1],zmm2[3],k1[3],zmm2[5],k1[5],zmm2[7],k1[7] +; CHECK-NEXT: vunpckhpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] +; CHECK-NEXT: vaddpd %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.unpckh.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3) + %res1 = call <8 x double> @llvm.x86.avx512.mask.unpckh.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1) + %res2 = fadd <8 x double> %res, %res1 + ret <8 x double> %res2 +} + +declare <16 x float> @llvm.x86.avx512.mask.unpckh.ps.512(<16 x float>, <16 x float>, <16 x float>, i16) + +define <16 x float>@test_int_x86_avx512_mask_unpckh_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_unpckh_ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vunpckhps {{.*#+}} zmm2 = zmm2[2],k1[2],zmm2[3],k1[3],zmm2[6],k1[6],zmm2[7],k1[7],zmm2[10],k1[10],zmm2[11],k1[11],zmm2[14],k1[14],zmm2[15],k1[15] +; CHECK-NEXT: vunpckhps {{.*#+}} zmm0 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] +; CHECK-NEXT: vaddps %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.unpckh.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3) + %res1 = call <16 x float> @llvm.x86.avx512.mask.unpckh.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1) + %res2 = fadd <16 x float> %res, %res1 + ret <16 x float> %res2 +} + +declare <8 x double> @llvm.x86.avx512.mask.unpckl.pd.512(<8 x double>, <8 x double>, <8 x double>, i8) + +define <8 x double>@test_int_x86_avx512_mask_unpckl_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_unpckl_pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vunpcklpd {{.*#+}} zmm2 = zmm2[0],k1[0],zmm2[2],k1[2],zmm2[4],k1[4],zmm2[6],k1[6] +; CHECK-NEXT: vunpcklpd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] +; CHECK-NEXT: vaddpd %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.unpckl.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3) + %res1 = call <8 x double> @llvm.x86.avx512.mask.unpckl.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1) + %res2 = fadd <8 x double> %res, %res1 + ret <8 x double> %res2 +} + +declare <16 x float> @llvm.x86.avx512.mask.unpckl.ps.512(<16 x float>, <16 x float>, <16 x float>, i16) + +define <16 x float>@test_int_x86_avx512_mask_unpckl_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_unpckl_ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vunpcklps {{.*#+}} zmm2 = zmm2[0],k1[0],zmm2[1],k1[1],zmm2[4],k1[4],zmm2[5],k1[5],zmm2[8],k1[8],zmm2[9],k1[9],zmm2[12],k1[12],zmm2[13],k1[13] +; CHECK-NEXT: vunpcklps {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] +; CHECK-NEXT: vaddps %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.unpckl.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3) + %res1 = call <16 x float> @llvm.x86.avx512.mask.unpckl.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1) + %res2 = fadd <16 x float> %res, %res1 + ret <16 x float> %res2 +} + +declare <8 x i64> @llvm.x86.avx512.mask.punpcklqd.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8) + +define <8 x i64>@test_int_x86_avx512_mask_punpcklqd_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpcklqd_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpunpcklqdq {{.*#+}} zmm2 = zmm2[0],k1[0],zmm2[2],k1[2],zmm2[4],k1[4],zmm2[6],k1[6] +; CHECK-NEXT: vpunpcklqdq {{.*#+}} zmm3 = k1[0],zmm0[0],k1[2],zmm0[2],k1[4],zmm0[4],k1[6],zmm0[6] +; CHECK-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] +; CHECK-NEXT: vpaddq %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm3, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.punpcklqd.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) + %res1 = call <8 x i64> @llvm.x86.avx512.mask.punpcklqd.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1) + %res2 = call <8 x i64> @llvm.x86.avx512.mask.punpcklqd.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer,i8 %x3) + %res3 = add <8 x i64> %res, %res1 + %res4 = add <8 x i64> %res2, %res3 + ret <8 x i64> %res4 +} + +declare <8 x i64> @llvm.x86.avx512.mask.punpckhqd.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8) + +define <8 x i64>@test_int_x86_avx512_mask_punpckhqd_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpckhqd_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpunpckhqdq {{.*#+}} zmm2 = zmm2[1],k1[1],zmm2[3],k1[3],zmm2[5],k1[5],zmm2[7],k1[7] +; CHECK-NEXT: vpunpckhqdq {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] +; CHECK-NEXT: vpaddq %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.punpckhqd.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) + %res1 = call <8 x i64> @llvm.x86.avx512.mask.punpckhqd.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1) + %res2 = add <8 x i64> %res, %res1 + ret <8 x i64> %res2 +} + +declare <16 x i32> @llvm.x86.avx512.mask.punpckhd.q.512(<16 x i32>, <16 x i32>, <16 x i32>, i16) + +define <16 x i32>@test_int_x86_avx512_mask_punpckhd_q_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpckhd_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm2[2],k1[2],zmm2[3],k1[3],zmm2[6],k1[6],zmm2[7],k1[7],zmm2[10],k1[10],zmm2[11],k1[11],zmm2[14],k1[14],zmm2[15],k1[15] +; CHECK-NEXT: vpunpckhdq {{.*#+}} zmm0 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] +; CHECK-NEXT: vpaddd %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x i32> @llvm.x86.avx512.mask.punpckhd.q.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) + %res1 = call <16 x i32> @llvm.x86.avx512.mask.punpckhd.q.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1) + %res2 = add <16 x i32> %res, %res1 + ret <16 x i32> %res2 +} + +declare <16 x i32> @llvm.x86.avx512.mask.punpckld.q.512(<16 x i32>, <16 x i32>, <16 x i32>, i16) + +define <16 x i32>@test_int_x86_avx512_mask_punpckld_q_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpckld_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpunpckldq {{.*#+}} zmm2 = zmm2[0],k1[0],zmm2[1],k1[1],zmm2[4],k1[4],zmm2[5],k1[5],zmm2[8],k1[8],zmm2[9],k1[9],zmm2[12],k1[12],zmm2[13],k1[13] +; CHECK-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] +; CHECK-NEXT: vpaddd %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x i32> @llvm.x86.avx512.mask.punpckld.q.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) + %res1 = call <16 x i32> @llvm.x86.avx512.mask.punpckld.q.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1) + %res2 = add <16 x i32> %res, %res1 + ret <16 x i32> %res2 +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmov_qb_512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qb_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpmovqb %zmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovqb %zmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovqb %zmm0, %xmm0 +; CHECK-NEXT: vpaddb %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddb %xmm2, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.qb.mem.512(i8* %ptr, <8 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmov_qb_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qb_mem_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmovqb %zmm0, (%rdi) +; CHECK-NEXT: vpmovqb %zmm0, (%rdi) {%k1} +; CHECK-NEXT: retq + call void @llvm.x86.avx512.mask.pmov.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmov.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmovs_qb_512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qb_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpmovsqb %zmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovsqb %zmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovsqb %zmm0, %xmm0 +; CHECK-NEXT: vpaddb %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddb %xmm2, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.qb.mem.512(i8* %ptr, <8 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovs_qb_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qb_mem_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovsqb %zmm0, (%rdi) +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpmovsqb %zmm0, (%rdi) {%k1} +; CHECK-NEXT: retq + call void @llvm.x86.avx512.mask.pmovs.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovs.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmovus_qb_512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qb_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpmovusqb %zmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovusqb %zmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovusqb %zmm0, %xmm0 +; CHECK-NEXT: vpaddb %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddb %xmm2, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.qb.mem.512(i8* %ptr, <8 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovus_qb_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qb_mem_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovusqb %zmm0, (%rdi) +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpmovusqb %zmm0, (%rdi) {%k1} +; CHECK-NEXT: retq + call void @llvm.x86.avx512.mask.pmovus.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovus.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2) + ret void +} + +declare <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_pmov_qw_512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qw_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmovqw %zmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovqw %zmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovqw %zmm0, %xmm0 +; CHECK-NEXT: vpaddw %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddw %xmm2, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 -1) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2) + %res3 = add <8 x i16> %res0, %res1 + %res4 = add <8 x i16> %res3, %res2 + ret <8 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.qw.mem.512(i8* %ptr, <8 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmov_qw_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qw_mem_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmovqw %zmm0, (%rdi) +; CHECK-NEXT: vpmovqw %zmm0, (%rdi) {%k1} +; CHECK-NEXT: retq + call void @llvm.x86.avx512.mask.pmov.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmov.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2) + ret void +} + +declare <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_pmovs_qw_512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qw_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmovsqw %zmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovsqw %zmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovsqw %zmm0, %xmm0 +; CHECK-NEXT: vpaddw %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddw %xmm2, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 -1) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2) + %res3 = add <8 x i16> %res0, %res1 + %res4 = add <8 x i16> %res3, %res2 + ret <8 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.qw.mem.512(i8* %ptr, <8 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovs_qw_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qw_mem_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovsqw %zmm0, (%rdi) +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpmovsqw %zmm0, (%rdi) {%k1} +; CHECK-NEXT: retq + call void @llvm.x86.avx512.mask.pmovs.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovs.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2) + ret void +} + +declare <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_pmovus_qw_512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qw_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmovusqw %zmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovusqw %zmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovusqw %zmm0, %xmm0 +; CHECK-NEXT: vpaddw %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddw %xmm2, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 -1) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2) + %res3 = add <8 x i16> %res0, %res1 + %res4 = add <8 x i16> %res3, %res2 + ret <8 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.qw.mem.512(i8* %ptr, <8 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovus_qw_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qw_mem_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovusqw %zmm0, (%rdi) +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpmovusqw %zmm0, (%rdi) {%k1} +; CHECK-NEXT: retq + call void @llvm.x86.avx512.mask.pmovus.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovus.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2) + ret void +} + +declare <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64>, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_mask_pmov_qd_512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmovqd %zmm0, %ymm1 {%k1} +; CHECK-NEXT: vpmovqd %zmm0, %ymm2 {%k1} {z} +; CHECK-NEXT: vpmovqd %zmm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm2, %ymm0, %ymm0 +; CHECK-NEXT: retq + %res0 = call <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 -1) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2) + %res2 = call <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64> %x0, <8 x i32> zeroinitializer, i8 %x2) + %res3 = add <8 x i32> %res0, %res1 + %res4 = add <8 x i32> %res3, %res2 + ret <8 x i32> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.qd.mem.512(i8* %ptr, <8 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmov_qd_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qd_mem_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmovqd %zmm0, (%rdi) +; CHECK-NEXT: vpmovqd %zmm0, (%rdi) {%k1} +; CHECK-NEXT: retq + call void @llvm.x86.avx512.mask.pmov.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmov.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2) + ret void +} + +declare <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64>, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_mask_pmovs_qd_512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmovsqd %zmm0, %ymm1 {%k1} +; CHECK-NEXT: vpmovsqd %zmm0, %ymm2 {%k1} {z} +; CHECK-NEXT: vpmovsqd %zmm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm2, %ymm0, %ymm0 +; CHECK-NEXT: retq + %res0 = call <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 -1) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2) + %res2 = call <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64> %x0, <8 x i32> zeroinitializer, i8 %x2) + %res3 = add <8 x i32> %res0, %res1 + %res4 = add <8 x i32> %res3, %res2 + ret <8 x i32> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.qd.mem.512(i8* %ptr, <8 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovs_qd_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qd_mem_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovsqd %zmm0, (%rdi) +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpmovsqd %zmm0, (%rdi) {%k1} +; CHECK-NEXT: retq + call void @llvm.x86.avx512.mask.pmovs.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovs.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2) + ret void +} + +declare <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64>, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_mask_pmovus_qd_512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmovusqd %zmm0, %ymm1 {%k1} +; CHECK-NEXT: vpmovusqd %zmm0, %ymm2 {%k1} {z} +; CHECK-NEXT: vpmovusqd %zmm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm2, %ymm0, %ymm0 +; CHECK-NEXT: retq + %res0 = call <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 -1) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2) + %res2 = call <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64> %x0, <8 x i32> zeroinitializer, i8 %x2) + %res3 = add <8 x i32> %res0, %res1 + %res4 = add <8 x i32> %res3, %res2 + ret <8 x i32> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.qd.mem.512(i8* %ptr, <8 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovus_qd_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qd_mem_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovusqd %zmm0, (%rdi) +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpmovusqd %zmm0, (%rdi) {%k1} +; CHECK-NEXT: retq + call void @llvm.x86.avx512.mask.pmovus.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovus.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32>, <16 x i8>, i16) + +define <16 x i8>@test_int_x86_avx512_mask_pmov_db_512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_db_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpmovdb %zmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovdb %zmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovdb %zmm0, %xmm0 +; CHECK-NEXT: vpaddb %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddb %xmm2, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32> %x0, <16 x i8> zeroinitializer, i16 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.db.mem.512(i8* %ptr, <16 x i32>, i16) + +define void @test_int_x86_avx512_mask_pmov_db_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_db_mem_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpmovdb %zmm0, (%rdi) +; CHECK-NEXT: vpmovdb %zmm0, (%rdi) {%k1} +; CHECK-NEXT: retq + call void @llvm.x86.avx512.mask.pmov.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1) + call void @llvm.x86.avx512.mask.pmov.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32>, <16 x i8>, i16) + +define <16 x i8>@test_int_x86_avx512_mask_pmovs_db_512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_db_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpmovsdb %zmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovsdb %zmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovsdb %zmm0, %xmm0 +; CHECK-NEXT: vpaddb %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddb %xmm2, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32> %x0, <16 x i8> zeroinitializer, i16 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.db.mem.512(i8* %ptr, <16 x i32>, i16) + +define void @test_int_x86_avx512_mask_pmovs_db_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_db_mem_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovsdb %zmm0, (%rdi) +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpmovsdb %zmm0, (%rdi) {%k1} +; CHECK-NEXT: retq + call void @llvm.x86.avx512.mask.pmovs.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1) + call void @llvm.x86.avx512.mask.pmovs.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32>, <16 x i8>, i16) + +define <16 x i8>@test_int_x86_avx512_mask_pmovus_db_512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_db_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpmovusdb %zmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovusdb %zmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovusdb %zmm0, %xmm0 +; CHECK-NEXT: vpaddb %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddb %xmm2, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32> %x0, <16 x i8> zeroinitializer, i16 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.db.mem.512(i8* %ptr, <16 x i32>, i16) + +define void @test_int_x86_avx512_mask_pmovus_db_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_db_mem_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovusdb %zmm0, (%rdi) +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpmovusdb %zmm0, (%rdi) {%k1} +; CHECK-NEXT: retq + call void @llvm.x86.avx512.mask.pmovus.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1) + call void @llvm.x86.avx512.mask.pmovus.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2) + ret void +} + +declare <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32>, <16 x i16>, i16) + +define <16 x i16>@test_int_x86_avx512_mask_pmov_dw_512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_dw_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpmovdw %zmm0, %ymm1 {%k1} +; CHECK-NEXT: vpmovdw %zmm0, %ymm2 {%k1} {z} +; CHECK-NEXT: vpmovdw %zmm0, %ymm0 +; CHECK-NEXT: vpaddw %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddw %ymm2, %ymm0, %ymm0 +; CHECK-NEXT: retq + %res0 = call <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 -1) + %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2) + %res2 = call <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32> %x0, <16 x i16> zeroinitializer, i16 %x2) + %res3 = add <16 x i16> %res0, %res1 + %res4 = add <16 x i16> %res3, %res2 + ret <16 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.dw.mem.512(i8* %ptr, <16 x i32>, i16) + +define void @test_int_x86_avx512_mask_pmov_dw_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_dw_mem_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpmovdw %zmm0, (%rdi) +; CHECK-NEXT: vpmovdw %zmm0, (%rdi) {%k1} +; CHECK-NEXT: retq + call void @llvm.x86.avx512.mask.pmov.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1) + call void @llvm.x86.avx512.mask.pmov.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2) + ret void +} + +declare <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32>, <16 x i16>, i16) + +define <16 x i16>@test_int_x86_avx512_mask_pmovs_dw_512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_dw_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpmovsdw %zmm0, %ymm1 {%k1} +; CHECK-NEXT: vpmovsdw %zmm0, %ymm2 {%k1} {z} +; CHECK-NEXT: vpmovsdw %zmm0, %ymm0 +; CHECK-NEXT: vpaddw %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddw %ymm2, %ymm0, %ymm0 +; CHECK-NEXT: retq + %res0 = call <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 -1) + %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2) + %res2 = call <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32> %x0, <16 x i16> zeroinitializer, i16 %x2) + %res3 = add <16 x i16> %res0, %res1 + %res4 = add <16 x i16> %res3, %res2 + ret <16 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.dw.mem.512(i8* %ptr, <16 x i32>, i16) + +define void @test_int_x86_avx512_mask_pmovs_dw_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_dw_mem_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovsdw %zmm0, (%rdi) +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpmovsdw %zmm0, (%rdi) {%k1} +; CHECK-NEXT: retq + call void @llvm.x86.avx512.mask.pmovs.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1) + call void @llvm.x86.avx512.mask.pmovs.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2) + ret void +} + +declare <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32>, <16 x i16>, i16) + +define <16 x i16>@test_int_x86_avx512_mask_pmovus_dw_512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_dw_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpmovusdw %zmm0, %ymm1 {%k1} +; CHECK-NEXT: vpmovusdw %zmm0, %ymm2 {%k1} {z} +; CHECK-NEXT: vpmovusdw %zmm0, %ymm0 +; CHECK-NEXT: vpaddw %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddw %ymm2, %ymm0, %ymm0 +; CHECK-NEXT: retq + %res0 = call <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 -1) + %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2) + %res2 = call <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32> %x0, <16 x i16> zeroinitializer, i16 %x2) + %res3 = add <16 x i16> %res0, %res1 + %res4 = add <16 x i16> %res3, %res2 + ret <16 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.dw.mem.512(i8* %ptr, <16 x i32>, i16) + +define void @test_int_x86_avx512_mask_pmovus_dw_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_dw_mem_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovusdw %zmm0, (%rdi) +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpmovusdw %zmm0, (%rdi) {%k1} +; CHECK-NEXT: retq + call void @llvm.x86.avx512.mask.pmovus.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1) + call void @llvm.x86.avx512.mask.pmovus.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2) + ret void +} + +declare <8 x double> @llvm.x86.avx512.mask.cvtdq2pd.512(<8 x i32>, <8 x double>, i8) + +define <8 x double>@test_int_x86_avx512_mask_cvt_dq2pd_512(<8 x i32> %x0, <8 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_dq2pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtdq2pd %ymm0, %zmm1 {%k1} +; CHECK-NEXT: vcvtdq2pd %ymm0, %zmm0 +; CHECK-NEXT: vaddpd %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.cvtdq2pd.512(<8 x i32> %x0, <8 x double> %x1, i8 %x2) + %res1 = call <8 x double> @llvm.x86.avx512.mask.cvtdq2pd.512(<8 x i32> %x0, <8 x double> %x1, i8 -1) + %res2 = fadd <8 x double> %res, %res1 + ret <8 x double> %res2 +} + +declare <16 x float> @llvm.x86.avx512.mask.cvtdq2ps.512(<16 x i32>, <16 x float>, i16, i32) + +define <16 x float>@test_int_x86_avx512_mask_cvt_dq2ps_512(<16 x i32> %x0, <16 x float> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_dq2ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vcvtdq2ps %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vcvtdq2ps {rn-sae}, %zmm0, %zmm0 +; CHECK-NEXT: vaddps %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.cvtdq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 %x2, i32 4) + %res1 = call <16 x float> @llvm.x86.avx512.mask.cvtdq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 -1, i32 0) + %res2 = fadd <16 x float> %res, %res1 + ret <16 x float> %res2 +} + +declare <8 x i32> @llvm.x86.avx512.mask.cvtpd2dq.512(<8 x double>, <8 x i32>, i8, i32) + +define <8 x i32>@test_int_x86_avx512_mask_cvt_pd2dq_512(<8 x double> %x0, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2dq_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtpd2dq %zmm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtpd2dq {rn-sae}, %zmm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2dq.512(<8 x double> %x0, <8 x i32> %x1, i8 %x2, i32 4) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2dq.512(<8 x double> %x0, <8 x i32> %x1, i8 -1, i32 0) + %res2 = add <8 x i32> %res, %res1 + ret <8 x i32> %res2 +} + +declare <8 x float> @llvm.x86.avx512.mask.cvtpd2ps.512(<8 x double>, <8 x float>, i8, i32) + +define <8 x float>@test_int_x86_avx512_mask_cvt_pd2ps_512(<8 x double> %x0, <8 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtpd2ps %zmm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtpd2ps {ru-sae}, %zmm0, %ymm0 +; CHECK-NEXT: vaddps %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.avx512.mask.cvtpd2ps.512(<8 x double> %x0, <8 x float> %x1, i8 %x2, i32 4) + %res1 = call <8 x float> @llvm.x86.avx512.mask.cvtpd2ps.512(<8 x double> %x0, <8 x float> %x1, i8 -1, i32 2) + %res2 = fadd <8 x float> %res, %res1 + ret <8 x float> %res2 +} + +declare <8 x i32> @llvm.x86.avx512.mask.cvtpd2udq.512(<8 x double>, <8 x i32>, i8, i32) + +define <8 x i32>@test_int_x86_avx512_mask_cvt_pd2udq_512(<8 x double> %x0, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2udq_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtpd2udq {ru-sae}, %zmm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtpd2udq {rn-sae}, %zmm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2udq.512(<8 x double> %x0, <8 x i32> %x1, i8 %x2, i32 2) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2udq.512(<8 x double> %x0, <8 x i32> %x1, i8 -1, i32 0) + %res2 = add <8 x i32> %res, %res1 + ret <8 x i32> %res2 +} + +declare <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float>, <16 x i32>, i16, i32) + +define <16 x i32>@test_int_x86_avx512_mask_cvt_ps2dq_512(<16 x float> %x0, <16 x i32> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2dq_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vcvtps2dq {ru-sae}, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vcvtps2dq {rn-sae}, %zmm0, %zmm0 +; CHECK-NEXT: vpaddd %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float> %x0, <16 x i32> %x1, i16 %x2, i32 2) + %res1 = call <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float> %x0, <16 x i32> %x1, i16 -1, i32 0) + %res2 = add <16 x i32> %res, %res1 + ret <16 x i32> %res2 +} + +declare <8 x double> @llvm.x86.avx512.mask.cvtps2pd.512(<8 x float>, <8 x double>, i8, i32) + +define <8 x double>@test_int_x86_avx512_mask_cvt_ps2pd_512(<8 x float> %x0, <8 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtps2pd %ymm0, %zmm1 {%k1} +; CHECK-NEXT: vcvtps2pd {sae}, %ymm0, %zmm0 +; CHECK-NEXT: vaddpd %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.cvtps2pd.512(<8 x float> %x0, <8 x double> %x1, i8 %x2, i32 4) + %res1 = call <8 x double> @llvm.x86.avx512.mask.cvtps2pd.512(<8 x float> %x0, <8 x double> %x1, i8 -1, i32 8) + %res2 = fadd <8 x double> %res, %res1 + ret <8 x double> %res2 +} + +declare <16 x i32> @llvm.x86.avx512.mask.cvtps2udq.512(<16 x float>, <16 x i32>, i16, i32) + +define <16 x i32>@test_int_x86_avx512_mask_cvt_ps2udq_512(<16 x float> %x0, <16 x i32> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2udq_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vcvtps2udq {ru-sae}, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vcvtps2udq {rn-sae}, %zmm0, %zmm0 +; CHECK-NEXT: vpaddd %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x i32> @llvm.x86.avx512.mask.cvtps2udq.512(<16 x float> %x0, <16 x i32> %x1, i16 %x2, i32 2) + %res1 = call <16 x i32> @llvm.x86.avx512.mask.cvtps2udq.512(<16 x float> %x0, <16 x i32> %x1, i16 -1, i32 0) + %res2 = add <16 x i32> %res, %res1 + ret <16 x i32> %res2 +} + +declare <8 x i32> @llvm.x86.avx512.mask.cvttpd2dq.512(<8 x double>, <8 x i32>, i8, i32) + +define <8 x i32>@test_int_x86_avx512_mask_cvtt_pd2dq_512(<8 x double> %x0, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2dq_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvttpd2dq %zmm0, %ymm1 {%k1} +; CHECK-NEXT: vcvttpd2dq {sae}, %zmm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2dq.512(<8 x double> %x0, <8 x i32> %x1, i8 %x2, i32 4) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2dq.512(<8 x double> %x0, <8 x i32> %x1, i8 -1, i32 8) + %res2 = add <8 x i32> %res, %res1 + ret <8 x i32> %res2 +} + +declare <8 x double> @llvm.x86.avx512.mask.cvtudq2pd.512(<8 x i32>, <8 x double>, i8) + +define <8 x double>@test_int_x86_avx512_mask_cvt_udq2pd_512(<8 x i32> %x0, <8 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_udq2pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtudq2pd %ymm0, %zmm1 {%k1} +; CHECK-NEXT: vcvtudq2pd %ymm0, %zmm0 +; CHECK-NEXT: vaddpd %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.cvtudq2pd.512(<8 x i32> %x0, <8 x double> %x1, i8 %x2) + %res1 = call <8 x double> @llvm.x86.avx512.mask.cvtudq2pd.512(<8 x i32> %x0, <8 x double> %x1, i8 -1) + %res2 = fadd <8 x double> %res, %res1 + ret <8 x double> %res2 +} + + +declare <16 x float> @llvm.x86.avx512.mask.cvtudq2ps.512(<16 x i32>, <16 x float>, i16, i32) + +define <16 x float>@test_int_x86_avx512_mask_cvt_udq2ps_512(<16 x i32> %x0, <16 x float> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_udq2ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vcvtudq2ps %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vcvtudq2ps {rn-sae}, %zmm0, %zmm0 +; CHECK-NEXT: vaddps %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.cvtudq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 %x2, i32 4) + %res1 = call <16 x float> @llvm.x86.avx512.mask.cvtudq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 -1, i32 0) + %res2 = fadd <16 x float> %res, %res1 + ret <16 x float> %res2 +} + +declare <8 x i32> @llvm.x86.avx512.mask.cvttpd2udq.512(<8 x double>, <8 x i32>, i8, i32) + +define <8 x i32>@test_int_x86_avx512_mask_cvtt_pd2udq_512(<8 x double> %x0, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2udq_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvttpd2udq %zmm0, %ymm1 {%k1} +; CHECK-NEXT: vcvttpd2udq {sae}, %zmm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2udq.512(<8 x double> %x0, <8 x i32> %x1, i8 %x2, i32 4) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2udq.512(<8 x double> %x0, <8 x i32> %x1, i8 -1, i32 8) + %res2 = add <8 x i32> %res, %res1 + ret <8 x i32> %res2 +} + +declare <16 x i32> @llvm.x86.avx512.mask.cvttps2dq.512(<16 x float>, <16 x i32>, i16, i32) + +define <16 x i32>@test_int_x86_avx512_mask_cvtt_ps2dq_512(<16 x float> %x0, <16 x i32> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2dq_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vcvttps2dq %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vcvttps2dq {sae}, %zmm0, %zmm0 +; CHECK-NEXT: vpaddd %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x i32> @llvm.x86.avx512.mask.cvttps2dq.512(<16 x float> %x0, <16 x i32> %x1, i16 %x2, i32 4) + %res1 = call <16 x i32> @llvm.x86.avx512.mask.cvttps2dq.512(<16 x float> %x0, <16 x i32> %x1, i16 -1, i32 8) + %res2 = add <16 x i32> %res, %res1 + ret <16 x i32> %res2 +} + +declare <16 x i32> @llvm.x86.avx512.mask.cvttps2udq.512(<16 x float>, <16 x i32>, i16, i32) + +define <16 x i32>@test_int_x86_avx512_mask_cvtt_ps2udq_512(<16 x float> %x0, <16 x i32> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2udq_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vcvttps2udq %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vcvttps2udq {sae}, %zmm0, %zmm0 +; CHECK-NEXT: vpaddd %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x i32> @llvm.x86.avx512.mask.cvttps2udq.512(<16 x float> %x0, <16 x i32> %x1, i16 %x2, i32 4) + %res1 = call <16 x i32> @llvm.x86.avx512.mask.cvttps2udq.512(<16 x float> %x0, <16 x i32> %x1, i16 -1, i32 8) + %res2 = add <16 x i32> %res, %res1 + ret <16 x i32> %res2 +} + + +declare <4 x float> @llvm.x86.avx512.mask.scalef.ss(<4 x float>, <4 x float>,<4 x float>, i8, i32) +define <4 x float>@test_int_x86_avx512_mask_scalef_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_scalef_ss: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vscalefss %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vscalefss {rn-sae}, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vaddps %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.scalef.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 %x4, i32 4) + %res1 = call <4 x float> @llvm.x86.avx512.mask.scalef.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 -1, i32 8) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <2 x double> @llvm.x86.avx512.mask.scalef.sd(<2 x double>, <2 x double>,<2 x double>, i8, i32) +define <2 x double>@test_int_x86_avx512_mask_scalef_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_scalef_sd: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vscalefsd %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vscalefsd {rn-sae}, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vaddpd %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.scalef.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 %x4, i32 4) + %res1 = call <2 x double> @llvm.x86.avx512.mask.scalef.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 -1, i32 8) + %res2 = fadd <2 x double> %res, %res1 + ret <2 x double> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone + +define <4 x float> @test_getexp_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) { +; CHECK-LABEL: test_getexp_ss: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovaps %zmm2, %zmm3 +; CHECK-NEXT: vgetexpss %xmm1, %xmm0, %xmm3 {%k1} +; CHECK-NEXT: vgetexpss {sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vgetexpss {sae}, %xmm1, %xmm0, %xmm4 {%k1} {z} +; CHECK-NEXT: vgetexpss {sae}, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vaddps %xmm2, %xmm3, %xmm1 +; CHECK-NEXT: vaddps %xmm0, %xmm4, %xmm0 +; CHECK-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res0 = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 4) + %res1 = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 8) + %res2 = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 8) + %res3 = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 8) + + %res.1 = fadd <4 x float> %res0, %res1 + %res.2 = fadd <4 x float> %res2, %res3 + %res = fadd <4 x float> %res.1, %res.2 + ret <4 x float> %res +} + +declare <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone + +define <2 x double> @test_getexp_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) { +; CHECK-LABEL: test_getexp_sd: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovaps %zmm2, %zmm3 +; CHECK-NEXT: vgetexpsd %xmm1, %xmm0, %xmm3 {%k1} +; CHECK-NEXT: vgetexpsd %xmm1, %xmm0, %xmm4 +; CHECK-NEXT: vgetexpsd {sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vgetexpsd {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z} +; CHECK-NEXT: vaddpd %xmm2, %xmm3, %xmm1 +; CHECK-NEXT: vaddpd %xmm4, %xmm0, %xmm0 +; CHECK-NEXT: vaddpd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res0 = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 4) + %res1 = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 8) + %res2 = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 8) + %res3 = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 4) + + %res.1 = fadd <2 x double> %res0, %res1 + %res.2 = fadd <2 x double> %res2, %res3 + %res = fadd <2 x double> %res.1, %res.2 + ret <2 x double> %res +} + +declare i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double>, <2 x double>, i32, i8, i32) + +define i8@test_int_x86_avx512_mask_cmp_sd(<2 x double> %x0, <2 x double> %x1, i8 %x3, i32 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_cmp_sd: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vcmpnltsd {sae}, %xmm1, %xmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: shlb $7, %al +; CHECK-NEXT: sarb $7, %al +; CHECK-NEXT: retq + + %res4 = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> %x0, <2 x double> %x1, i32 5, i8 %x3, i32 8) + ret i8 %res4 +} + +define i8@test_int_x86_avx512_mask_cmp_sd_all(<2 x double> %x0, <2 x double> %x1, i8 %x3, i32 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_cmp_sd_all: +; CHECK: ## BB#0: +; CHECK-NEXT: vcmpunordsd {sae}, %xmm1, %xmm0, %k0 +; CHECK-NEXT: vcmplesd %xmm1, %xmm0, %k1 +; CHECK-NEXT: korw %k0, %k1, %k0 +; CHECK-NEXT: vcmpnltsd {sae}, %xmm1, %xmm0, %k1 +; CHECK-NEXT: vcmpneqsd %xmm1, %xmm0, %k2 +; CHECK-NEXT: korw %k1, %k2, %k1 +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k2 +; CHECK-NEXT: kandw %k2, %k1, %k1 +; CHECK-NEXT: korw %k1, %k0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: shlb $7, %al +; CHECK-NEXT: sarb $7, %al +; CHECK-NEXT: retq + + %res1 = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> %x0, <2 x double> %x1, i32 2, i8 -1, i32 4) + %res2 = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> %x0, <2 x double> %x1, i32 3, i8 -1, i32 8) + %res3 = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> %x0, <2 x double> %x1, i32 4, i8 %x3, i32 4) + %res4 = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> %x0, <2 x double> %x1, i32 5, i8 %x3, i32 8) + + %res11 = or i8 %res1, %res2 + %res12 = or i8 %res3, %res4 + %res13 = or i8 %res11, %res12 + ret i8 %res13 +} + +declare i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float>, <4 x float>, i32, i8, i32) + +define i8@test_int_x86_avx512_mask_cmp_ss(<4 x float> %x0, <4 x float> %x1, i8 %x3, i32 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_cmp_ss: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vcmpunordss %xmm1, %xmm0, %k0 {%k1} +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: shlb $7, %al +; CHECK-NEXT: sarb $7, %al +; CHECK-NEXT: retq + + %res2 = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> %x0, <4 x float> %x1, i32 3, i8 %x3, i32 4) + ret i8 %res2 +} + + +define i8@test_int_x86_avx512_mask_cmp_ss_all(<4 x float> %x0, <4 x float> %x1, i8 %x3, i32 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_cmp_ss_all: +; CHECK: ## BB#0: +; CHECK-NEXT: vcmpless %xmm1, %xmm0, %k1 +; CHECK-NEXT: vcmpunordss {sae}, %xmm1, %xmm0, %k0 {%k1} +; CHECK-NEXT: vcmpneqss %xmm1, %xmm0, %k1 +; CHECK-NEXT: vcmpnltss {sae}, %xmm1, %xmm0, %k1 {%k1} +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k2 +; CHECK-NEXT: kandw %k2, %k1, %k1 +; CHECK-NEXT: kandw %k1, %k0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: shlb $7, %al +; CHECK-NEXT: sarb $7, %al +; CHECK-NEXT: retq + %res1 = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> %x0, <4 x float> %x1, i32 2, i8 -1, i32 4) + %res2 = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> %x0, <4 x float> %x1, i32 3, i8 -1, i32 8) + %res3 = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> %x0, <4 x float> %x1, i32 4, i8 %x3, i32 4) + %res4 = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> %x0, <4 x float> %x1, i32 5, i8 %x3, i32 8) + + %res11 = and i8 %res1, %res2 + %res12 = and i8 %res3, %res4 + %res13 = and i8 %res11, %res12 + ret i8 %res13 +} + +declare <16 x float> @llvm.x86.avx512.mask.shuf.f32x4(<16 x float>, <16 x float>, i32, <16 x float>, i16) + +define <16 x float>@test_int_x86_avx512_mask_shuf_f32x4(<16 x float> %x0, <16 x float> %x1, <16 x float> %x3, i16 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_shuf_f32x4: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vshuff32x4 {{.*#+}} zmm2 = zmm0[8,9,10,11,4,5,6,7],zmm1[4,5,6,7,0,1,2,3] +; CHECK-NEXT: vshuff32x4 {{.*#+}} zmm0 = zmm0[8,9,10,11,4,5,6,7],zmm1[4,5,6,7,0,1,2,3] +; CHECK-NEXT: vaddps %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.shuf.f32x4(<16 x float> %x0, <16 x float> %x1, i32 22, <16 x float> %x3, i16 %x4) + %res1 = call <16 x float> @llvm.x86.avx512.mask.shuf.f32x4(<16 x float> %x0, <16 x float> %x1, i32 22, <16 x float> %x3, i16 -1) + %res2 = fadd <16 x float> %res, %res1 + ret <16 x float> %res2 +} + +declare <8 x double> @llvm.x86.avx512.mask.shuf.f64x2(<8 x double>, <8 x double>, i32, <8 x double>, i8) + +define <8 x double>@test_int_x86_avx512_mask_shuf_f64x2(<8 x double> %x0, <8 x double> %x1, <8 x double> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_shuf_f64x2: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vshuff64x2 {{.*#+}} zmm2 = zmm0[4,5,2,3],zmm1[2,3,0,1] +; CHECK-NEXT: vshuff64x2 {{.*#+}} zmm3 = zmm0[4,5,2,3],zmm1[2,3,0,1] +; CHECK-NEXT: vshuff64x2 {{.*#+}} zmm0 = zmm0[4,5,2,3],zmm1[2,3,0,1] +; CHECK-NEXT: vaddpd %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: vaddpd %zmm3, %zmm0, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.shuf.f64x2(<8 x double> %x0, <8 x double> %x1, i32 22, <8 x double> %x3, i8 %x4) + %res1 = call <8 x double> @llvm.x86.avx512.mask.shuf.f64x2(<8 x double> %x0, <8 x double> %x1, i32 22, <8 x double> %x3, i8 -1) + %res2 = call <8 x double> @llvm.x86.avx512.mask.shuf.f64x2(<8 x double> %x0, <8 x double> %x1, i32 22, <8 x double> zeroinitializer, i8 %x4) + + %res3 = fadd <8 x double> %res, %res1 + %res4 = fadd <8 x double> %res3, %res2 + ret <8 x double> %res4 +} + +declare <16 x i32> @llvm.x86.avx512.mask.shuf.i32x4(<16 x i32>, <16 x i32>, i32, <16 x i32>, i16) + +define <16 x i32>@test_int_x86_avx512_mask_shuf_i32x4(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x3, i16 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_shuf_i32x4: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vshufi32x4 {{.*#+}} zmm2 = zmm0[8,9,10,11,4,5,6,7],zmm1[4,5,6,7,0,1,2,3] +; CHECK-NEXT: vshufi32x4 {{.*#+}} zmm0 = zmm0[8,9,10,11,4,5,6,7],zmm1[4,5,6,7,0,1,2,3] +; CHECK-NEXT: vpaddd %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x i32> @llvm.x86.avx512.mask.shuf.i32x4(<16 x i32> %x0, <16 x i32> %x1, i32 22, <16 x i32> %x3, i16 %x4) + %res1 = call <16 x i32> @llvm.x86.avx512.mask.shuf.i32x4(<16 x i32> %x0, <16 x i32> %x1, i32 22, <16 x i32> %x3, i16 -1) + %res2 = add <16 x i32> %res, %res1 + ret <16 x i32> %res2 +} + +declare <8 x i64> @llvm.x86.avx512.mask.shuf.i64x2(<8 x i64>, <8 x i64>, i32, <8 x i64>, i8) + +define <8 x i64>@test_int_x86_avx512_mask_shuf_i64x2(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_shuf_i64x2: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm0[4,5,2,3],zmm1[2,3,0,1] +; CHECK-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[4,5,2,3],zmm1[2,3,0,1] +; CHECK-NEXT: vpaddq %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.shuf.i64x2(<8 x i64> %x0, <8 x i64> %x1, i32 22, <8 x i64> %x3, i8 %x4) + %res1 = call <8 x i64> @llvm.x86.avx512.mask.shuf.i64x2(<8 x i64> %x0, <8 x i64> %x1, i32 22, <8 x i64> %x3, i8 -1) + %res2 = add <8 x i64> %res, %res1 + ret <8 x i64> %res2 +} + +declare <8 x double> @llvm.x86.avx512.mask.getmant.pd.512(<8 x double>, i32, <8 x double>, i8, i32) + +define <8 x double>@test_int_x86_avx512_mask_getmant_pd_512(<8 x double> %x0, <8 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_getmant_pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vgetmantpd $11, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vgetmantpd $11,{sae}, %zmm0, %zmm0 +; CHECK-NEXT: vaddpd %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.getmant.pd.512(<8 x double> %x0, i32 11, <8 x double> %x2, i8 %x3, i32 4) + %res1 = call <8 x double> @llvm.x86.avx512.mask.getmant.pd.512(<8 x double> %x0, i32 11, <8 x double> %x2, i8 -1, i32 8) + %res2 = fadd <8 x double> %res, %res1 + ret <8 x double> %res2 +} + +declare <16 x float> @llvm.x86.avx512.mask.getmant.ps.512(<16 x float>, i32, <16 x float>, i16, i32) + +define <16 x float>@test_int_x86_avx512_mask_getmant_ps_512(<16 x float> %x0, <16 x float> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_getmant_ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vgetmantps $11, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vgetmantps $11,{sae}, %zmm0, %zmm0 +; CHECK-NEXT: vaddps %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.getmant.ps.512(<16 x float> %x0, i32 11, <16 x float> %x2, i16 %x3, i32 4) + %res1 = call <16 x float> @llvm.x86.avx512.mask.getmant.ps.512(<16 x float> %x0, i32 11, <16 x float> %x2, i16 -1, i32 8) + %res2 = fadd <16 x float> %res, %res1 + ret <16 x float> %res2 +} + +declare <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double>, <2 x double>, i32, <2 x double>, i8, i32) + +define <2 x double>@test_int_x86_avx512_mask_getmant_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_getmant_sd: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovaps %zmm2, %zmm3 +; CHECK-NEXT: vgetmantsd $11, %xmm1, %xmm0, %xmm3 {%k1} +; CHECK-NEXT: vgetmantsd $11, %xmm1, %xmm0, %xmm4 {%k1} {z} +; CHECK-NEXT: vgetmantsd $11, %xmm1, %xmm0, %xmm5 +; CHECK-NEXT: vgetmantsd $11,{sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vaddpd %xmm4, %xmm3, %xmm0 +; CHECK-NEXT: vaddpd %xmm5, %xmm2, %xmm1 +; CHECK-NEXT: vaddpd %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 11, <2 x double> %x2, i8 %x3, i32 4) + %res1 = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 11, <2 x double> zeroinitializer, i8 %x3, i32 4) + %res2 = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 11, <2 x double> %x2, i8 %x3, i32 8) + %res3 = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 11, <2 x double> %x2, i8 -1, i32 4) + %res11 = fadd <2 x double> %res, %res1 + %res12 = fadd <2 x double> %res2, %res3 + %res13 = fadd <2 x double> %res11, %res12 + ret <2 x double> %res13 +} + +declare <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float>, <4 x float>, i32, <4 x float>, i8, i32) + +define <4 x float>@test_int_x86_avx512_mask_getmant_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_getmant_ss: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vgetmantss $11, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vgetmantss $11, %xmm1, %xmm0, %xmm3 {%k1} {z} +; CHECK-NEXT: vgetmantss $11, %xmm1, %xmm0, %xmm4 +; CHECK-NEXT: vgetmantss $11,{sae}, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vaddps %xmm3, %xmm2, %xmm1 +; CHECK-NEXT: vaddps %xmm4, %xmm0, %xmm0 +; CHECK-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 11, <4 x float> %x2, i8 %x3, i32 4) + %res1 = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 11, <4 x float> zeroinitializer, i8 %x3, i32 4) + %res2 = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 11, <4 x float> %x2, i8 -1, i32 8) + %res3 = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 11, <4 x float> %x2, i8 -1, i32 4) + %res11 = fadd <4 x float> %res, %res1 + %res12 = fadd <4 x float> %res2, %res3 + %res13 = fadd <4 x float> %res11, %res12 + ret <4 x float> %res13 +} + +declare <8 x double> @llvm.x86.avx512.mask.shuf.pd.512(<8 x double>, <8 x double>, i32, <8 x double>, i8) + +define <8 x double>@test_int_x86_avx512_mask_shuf_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_shuf_pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vshufpd {{.*#+}} zmm2 = zmm2[0],k1[1],zmm2[3],k1[2],zmm2[5],k1[4],zmm2[6],k1[6] +; CHECK-NEXT: vshufpd {{.*#+}} zmm3 = k1[0],zmm0[1],k1[3],zmm0[2],k1[5],zmm0[4],k1[6],zmm0[6] +; CHECK-NEXT: vshufpd {{.*#+}} zmm0 = zmm0[0],zmm1[1],zmm0[3],zmm1[2],zmm0[5],zmm1[4],zmm0[6],zmm1[6] +; CHECK-NEXT: vaddpd %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: vaddpd %zmm3, %zmm0, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.shuf.pd.512(<8 x double> %x0, <8 x double> %x1, i32 22, <8 x double> %x3, i8 %x4) + %res1 = call <8 x double> @llvm.x86.avx512.mask.shuf.pd.512(<8 x double> %x0, <8 x double> %x1, i32 22, <8 x double> %x3, i8 -1) + %res2 = call <8 x double> @llvm.x86.avx512.mask.shuf.pd.512(<8 x double> %x0, <8 x double> %x1, i32 22, <8 x double> zeroinitializer, i8 %x4) + + %res3 = fadd <8 x double> %res, %res1 + %res4 = fadd <8 x double> %res3, %res2 + ret <8 x double> %res4 +} + +declare <16 x float> @llvm.x86.avx512.mask.shuf.ps.512(<16 x float>, <16 x float>, i32, <16 x float>, i16) + +define <16 x float>@test_int_x86_avx512_mask_shuf_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x3, i16 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_shuf_ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vshufps {{.*#+}} zmm2 = zmm2[2,1],k1[1,0],zmm2[6,5],k1[5,4],zmm2[10,9],k1[9,8],zmm2[14,13],k1[13,12] +; CHECK-NEXT: vshufps {{.*#+}} zmm0 = zmm0[2,1],zmm1[1,0],zmm0[6,5],zmm1[5,4],zmm0[10,9],zmm1[9,8],zmm0[14,13],zmm1[13,12] +; CHECK-NEXT: vaddps %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.shuf.ps.512(<16 x float> %x0, <16 x float> %x1, i32 22, <16 x float> %x3, i16 %x4) + %res1 = call <16 x float> @llvm.x86.avx512.mask.shuf.ps.512(<16 x float> %x0, <16 x float> %x1, i32 22, <16 x float> %x3, i16 -1) + %res2 = fadd <16 x float> %res, %res1 + ret <16 x float> %res2 +} + +declare <8 x double> @llvm.x86.avx512.mask.vpermil.pd.512(<8 x double>, i32, <8 x double>, i8) + +define <8 x double>@test_int_x86_avx512_mask_vpermil_pd_512(<8 x double> %x0, <8 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpermil_pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpermilpd {{.*#+}} zmm1 = zmm1[0,1,3,2,5,4,6,6] +; CHECK-NEXT: vpermilpd {{.*#+}} zmm2 = k1[0,1,3,2,5,4,6,6] +; CHECK-NEXT: vpermilpd {{.*#+}} zmm0 = zmm0[0,1,3,2,5,4,6,6] +; CHECK-NEXT: vaddpd %zmm2, %zmm1, %zmm1 +; CHECK-NEXT: vaddpd %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.vpermil.pd.512(<8 x double> %x0, i32 22, <8 x double> %x2, i8 %x3) + %res1 = call <8 x double> @llvm.x86.avx512.mask.vpermil.pd.512(<8 x double> %x0, i32 22, <8 x double> zeroinitializer, i8 %x3) + %res2 = call <8 x double> @llvm.x86.avx512.mask.vpermil.pd.512(<8 x double> %x0, i32 22, <8 x double> %x2, i8 -1) + %res3 = fadd <8 x double> %res, %res1 + %res4 = fadd <8 x double> %res3, %res2 + ret <8 x double> %res4 +} + +declare <16 x float> @llvm.x86.avx512.mask.vpermil.ps.512(<16 x float>, i32, <16 x float>, i16) + +define <16 x float>@test_int_x86_avx512_mask_vpermil_ps_512(<16 x float> %x0, <16 x float> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpermil_ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpermilps {{.*#+}} zmm1 = zmm1[2,1,1,0,6,5,5,4,10,9,9,8,14,13,13,12] +; CHECK-NEXT: vpermilps {{.*#+}} zmm2 = k1[2,1,1,0,6,5,5,4,10,9,9,8,14,13,13,12] +; CHECK-NEXT: vpermilps {{.*#+}} zmm0 = zmm0[2,1,1,0,6,5,5,4,10,9,9,8,14,13,13,12] +; CHECK-NEXT: vaddps %zmm2, %zmm1, %zmm1 +; CHECK-NEXT: vaddps %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.vpermil.ps.512(<16 x float> %x0, i32 22, <16 x float> %x2, i16 %x3) + %res1 = call <16 x float> @llvm.x86.avx512.mask.vpermil.ps.512(<16 x float> %x0, i32 22, <16 x float> zeroinitializer, i16 %x3) + %res2 = call <16 x float> @llvm.x86.avx512.mask.vpermil.ps.512(<16 x float> %x0, i32 22, <16 x float> %x2, i16 -1) + %res3 = fadd <16 x float> %res, %res1 + %res4 = fadd <16 x float> %res3, %res2 + ret <16 x float> %res4 +} + +declare <8 x double> @llvm.x86.avx512.mask.vpermilvar.pd.512(<8 x double>, <8 x i64>, <8 x double>, i8) + +define <8 x double>@test_int_x86_avx512_mask_vpermilvar_pd_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpermilvar_pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpermilpd %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vpermilpd %zmm1, %zmm0, %zmm3 {%k1} {z} +; CHECK-NEXT: vpermilpd %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: vaddpd %zmm3, %zmm2, %zmm1 +; CHECK-NEXT: vaddpd %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.vpermilvar.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3) + %res1 = call <8 x double> @llvm.x86.avx512.mask.vpermilvar.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> zeroinitializer, i8 %x3) + %res2 = call <8 x double> @llvm.x86.avx512.mask.vpermilvar.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 -1) + %res3 = fadd <8 x double> %res, %res1 + %res4 = fadd <8 x double> %res2, %res3 + ret <8 x double> %res4 +} + +declare <16 x float> @llvm.x86.avx512.mask.vpermilvar.ps.512(<16 x float>, <16 x i32>, <16 x float>, i16) + +define <16 x float>@test_int_x86_avx512_mask_vpermilvar_ps_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpermilvar_ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpermilps %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vpermilps %zmm1, %zmm0, %zmm3 {%k1} {z} +; CHECK-NEXT: vpermilps %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: vaddps %zmm3, %zmm2, %zmm1 +; CHECK-NEXT: vaddps %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3) + %res1 = call <16 x float> @llvm.x86.avx512.mask.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> zeroinitializer, i16 %x3) + %res2 = call <16 x float> @llvm.x86.avx512.mask.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 -1) + %res3 = fadd <16 x float> %res, %res1 + %res4 = fadd <16 x float> %res2, %res3 + ret <16 x float> %res4 +} + +declare <16 x float> @llvm.x86.avx512.mask.insertf32x4.512(<16 x float>, <4 x float>, i32, <16 x float>, i8) + +define <16 x float>@test_int_x86_avx512_mask_insertf32x4_512(<16 x float> %x0, <4 x float> %x1, <16 x float> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_insertf32x4_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vinsertf32x4 $1, %xmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vinsertf32x4 $1, %xmm1, %zmm0, %zmm3 {%k1} {z} +; CHECK-NEXT: vinsertf32x4 $1, %xmm1, %zmm0, %zmm0 +; CHECK-NEXT: vaddps %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: vaddps %zmm0, %zmm3, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.insertf32x4.512(<16 x float> %x0, <4 x float> %x1, i32 1, <16 x float> %x3, i8 %x4) + %res1 = call <16 x float> @llvm.x86.avx512.mask.insertf32x4.512(<16 x float> %x0, <4 x float> %x1, i32 1, <16 x float> %x3, i8 -1) + %res2 = call <16 x float> @llvm.x86.avx512.mask.insertf32x4.512(<16 x float> %x0, <4 x float> %x1, i32 1, <16 x float> zeroinitializer, i8 %x4) + %res3 = fadd <16 x float> %res, %res1 + %res4 = fadd <16 x float> %res2, %res3 + ret <16 x float> %res4 +} + +declare <16 x i32> @llvm.x86.avx512.mask.inserti32x4.512(<16 x i32>, <4 x i32>, i32, <16 x i32>, i8) + +define <16 x i32>@test_int_x86_avx512_mask_inserti32x4_512(<16 x i32> %x0, <4 x i32> %x1, <16 x i32> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_inserti32x4_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm3 {%k1} {z} +; CHECK-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0 +; CHECK-NEXT: vpaddd %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: vpaddd %zmm0, %zmm3, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x i32> @llvm.x86.avx512.mask.inserti32x4.512(<16 x i32> %x0, <4 x i32> %x1, i32 1, <16 x i32> %x3, i8 %x4) + %res1 = call <16 x i32> @llvm.x86.avx512.mask.inserti32x4.512(<16 x i32> %x0, <4 x i32> %x1, i32 1, <16 x i32> %x3, i8 -1) + %res2 = call <16 x i32> @llvm.x86.avx512.mask.inserti32x4.512(<16 x i32> %x0, <4 x i32> %x1, i32 1, <16 x i32> zeroinitializer, i8 %x4) + %res3 = add <16 x i32> %res, %res1 + %res4 = add <16 x i32> %res2, %res3 + ret <16 x i32> %res4 +} + +declare <8 x double> @llvm.x86.avx512.mask.insertf64x4.512(<8 x double>, <4 x double>, i32, <8 x double>, i8) + +define <8 x double>@test_int_x86_avx512_mask_insertf64x4_512(<8 x double> %x0, <4 x double> %x1, <8 x double> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_insertf64x4_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm3 {%k1} {z} +; CHECK-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 +; CHECK-NEXT: vaddpd %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: vaddpd %zmm0, %zmm3, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.insertf64x4.512(<8 x double> %x0, <4 x double> %x1, i32 1, <8 x double> %x3, i8 %x4) + %res1 = call <8 x double> @llvm.x86.avx512.mask.insertf64x4.512(<8 x double> %x0, <4 x double> %x1, i32 1, <8 x double> %x3, i8 -1) + %res2 = call <8 x double> @llvm.x86.avx512.mask.insertf64x4.512(<8 x double> %x0, <4 x double> %x1, i32 1, <8 x double> zeroinitializer, i8 %x4) + %res3 = fadd <8 x double> %res, %res1 + %res4 = fadd <8 x double> %res2, %res3 + ret <8 x double> %res4 +} + +declare <8 x i64> @llvm.x86.avx512.mask.inserti64x4.512(<8 x i64>, <4 x i64>, i32, <8 x i64>, i8) + +define <8 x i64>@test_int_x86_avx512_mask_inserti64x4_512(<8 x i64> %x0, <4 x i64> %x1, <8 x i64> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_inserti64x4_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm3 {%k1} {z} +; CHECK-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm3, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.inserti64x4.512(<8 x i64> %x0, <4 x i64> %x1, i32 1, <8 x i64> %x3, i8 %x4) + %res1 = call <8 x i64> @llvm.x86.avx512.mask.inserti64x4.512(<8 x i64> %x0, <4 x i64> %x1, i32 1, <8 x i64> %x3, i8 -1) + %res2 = call <8 x i64> @llvm.x86.avx512.mask.inserti64x4.512(<8 x i64> %x0, <4 x i64> %x1, i32 1, <8 x i64> zeroinitializer, i8 %x4) + %res3 = add <8 x i64> %res, %res1 + %res4 = add <8 x i64> %res2, %res3 + ret <8 x i64> %res4 +} + +declare <2 x double> @llvm.x86.avx512.mask.cvtss2sd.round(<4 x float>, <4 x float>, <2 x double>, i8, i32) + +define <2 x double>@test_int_x86_avx512_mask_cvt_ss2sd_round(<4 x float> %x0,<4 x float> %x1, <2 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ss2sd_round: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vcvtss2sd %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vcvtss2sd {sae}, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vaddpd %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.cvtss2sd.round(<4 x float> %x0, <4 x float> %x1, <2 x double> %x2, i8 %x3, i32 4) + %res1 = call <2 x double> @llvm.x86.avx512.mask.cvtss2sd.round(<4 x float> %x0, <4 x float> %x1, <2 x double> %x2, i8 -1, i32 8) + %res2 = fadd <2 x double> %res, %res1 + ret <2 x double> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.cvtsd2ss.round(<2 x double>, <2 x double>, <4 x float>, i8, i32) + +define <4 x float>@test_int_x86_avx512_mask_cvt_sd2ss_round(<2 x double> %x0,<2 x double> %x1, <4 x float> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_sd2ss_round: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vcvtsd2ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vcvtsd2ss {rn-sae}, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vaddps %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.cvtsd2ss.round(<2 x double> %x0, <2 x double> %x1, <4 x float> %x2, i8 %x3, i32 3) + %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtsd2ss.round(<2 x double> %x0, <2 x double> %x1, <4 x float> %x2, i8 -1, i32 8) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <16 x i32> @llvm.x86.avx512.mask.pternlog.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i32, i16) + +define <16 x i32>@test_int_x86_avx512_mask_pternlog_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_pternlog_d_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovaps %zmm0, %zmm3 +; CHECK-NEXT: vpternlogd $33, %zmm2, %zmm1, %zmm3 {%k1} +; CHECK-NEXT: vpternlogd $33, %zmm2, %zmm1, %zmm0 +; CHECK-NEXT: vpaddd %zmm0, %zmm3, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x i32> @llvm.x86.avx512.mask.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33, i16 %x4) + %res1 = call <16 x i32> @llvm.x86.avx512.mask.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33, i16 -1) + %res2 = add <16 x i32> %res, %res1 + ret <16 x i32> %res2 +} + +declare <16 x i32> @llvm.x86.avx512.maskz.pternlog.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i32, i16) + +define <16 x i32>@test_int_x86_avx512_maskz_pternlog_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x4) { +; CHECK-LABEL: test_int_x86_avx512_maskz_pternlog_d_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovaps %zmm0, %zmm3 +; CHECK-NEXT: vpternlogd $33, %zmm2, %zmm1, %zmm3 {%k1} {z} +; CHECK-NEXT: vpternlogd $33, %zmm2, %zmm1, %zmm0 +; CHECK-NEXT: vpaddd %zmm0, %zmm3, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x i32> @llvm.x86.avx512.maskz.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33, i16 %x4) + %res1 = call <16 x i32> @llvm.x86.avx512.maskz.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33, i16 -1) + %res2 = add <16 x i32> %res, %res1 + ret <16 x i32> %res2 +} + +declare <8 x i64> @llvm.x86.avx512.mask.pternlog.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i32, i8) + +define <8 x i64>@test_int_x86_avx512_mask_pternlog_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_pternlog_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovaps %zmm0, %zmm3 +; CHECK-NEXT: vpternlogq $33, %zmm2, %zmm1, %zmm3 {%k1} +; CHECK-NEXT: vpternlogq $33, %zmm2, %zmm1, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm3, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33, i8 %x4) + %res1 = call <8 x i64> @llvm.x86.avx512.mask.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33, i8 -1) + %res2 = add <8 x i64> %res, %res1 + ret <8 x i64> %res2 +} + +declare <8 x i64> @llvm.x86.avx512.maskz.pternlog.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i32, i8) + +define <8 x i64>@test_int_x86_avx512_maskz_pternlog_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_maskz_pternlog_q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovaps %zmm0, %zmm3 +; CHECK-NEXT: vpternlogq $33, %zmm2, %zmm1, %zmm3 {%k1} {z} +; CHECK-NEXT: vpternlogq $33, %zmm2, %zmm1, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm3, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.maskz.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33, i8 %x4) + %res1 = call <8 x i64> @llvm.x86.avx512.maskz.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33, i8 -1) + %res2 = add <8 x i64> %res, %res1 + ret <8 x i64> %res2 +} + +declare <16 x float> @llvm.x86.avx512.mask.movsldup.512(<16 x float>, <16 x float>, i16) + +define <16 x float>@test_int_x86_avx512_mask_movsldup_512(<16 x float> %x0, <16 x float> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_movsldup_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovsldup {{.*#+}} zmm1 = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14] +; CHECK-NEXT: vmovsldup {{.*#+}} zmm2 = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14] +; CHECK-NEXT: vmovsldup {{.*#+}} zmm0 = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14] +; CHECK-NEXT: vaddps %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: vaddps %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.movsldup.512(<16 x float> %x0, <16 x float> %x1, i16 %x2) + %res1 = call <16 x float> @llvm.x86.avx512.mask.movsldup.512(<16 x float> %x0, <16 x float> %x1, i16 -1) + %res2 = call <16 x float> @llvm.x86.avx512.mask.movsldup.512(<16 x float> %x0, <16 x float> zeroinitializer, i16 %x2) + %res3 = fadd <16 x float> %res, %res1 + %res4 = fadd <16 x float> %res2, %res3 + ret <16 x float> %res4 +} + +declare <16 x float> @llvm.x86.avx512.mask.movshdup.512(<16 x float>, <16 x float>, i16) + +define <16 x float>@test_int_x86_avx512_mask_movshdup_512(<16 x float> %x0, <16 x float> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_movshdup_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovshdup {{.*#+}} zmm1 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15] +; CHECK-NEXT: vmovshdup {{.*#+}} zmm2 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15] +; CHECK-NEXT: vmovshdup {{.*#+}} zmm0 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15] +; CHECK-NEXT: vaddps %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: vaddps %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.movshdup.512(<16 x float> %x0, <16 x float> %x1, i16 %x2) + %res1 = call <16 x float> @llvm.x86.avx512.mask.movshdup.512(<16 x float> %x0, <16 x float> %x1, i16 -1) + %res2 = call <16 x float> @llvm.x86.avx512.mask.movshdup.512(<16 x float> %x0, <16 x float> zeroinitializer, i16 %x2) + %res3 = fadd <16 x float> %res, %res1 + %res4 = fadd <16 x float> %res2, %res3 + ret <16 x float> %res4 +} + +declare <8 x double> @llvm.x86.avx512.mask.movddup.512(<8 x double>, <8 x double>, i8) + +define <8 x double>@test_int_x86_avx512_mask_movddup_512(<8 x double> %x0, <8 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_movddup_512: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovddup {{.*#+}} zmm1 = zmm0[0,0,2,2,4,4,6,6] +; CHECK-NEXT: vmovddup {{.*#+}} zmm2 = zmm0[0,0,2,2,4,4,6,6] +; CHECK-NEXT: vmovddup {{.*#+}} zmm0 = zmm0[0,0,2,2,4,4,6,6] +; CHECK-NEXT: vaddpd %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: vaddpd %zmm0, %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.movddup.512(<8 x double> %x0, <8 x double> %x1, i8 %x2) + %res1 = call <8 x double> @llvm.x86.avx512.mask.movddup.512(<8 x double> %x0, <8 x double> %x1, i8 -1) + %res2 = call <8 x double> @llvm.x86.avx512.mask.movddup.512(<8 x double> %x0, <8 x double> zeroinitializer, i8 %x2) + %res3 = fadd <8 x double> %res, %res1 + %res4 = fadd <8 x double> %res2, %res3 + ret <8 x double> %res4 +} + +define i32 @test_x86_avx512_comi_sd_eq_sae(<2 x double> %a0, <2 x double> %a1) { +; CHECK-LABEL: test_x86_avx512_comi_sd_eq_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vcomisd {sae}, %xmm1, %xmm0 +; CHECK-NEXT: sete %al +; CHECK-NEXT: movzbl %al, %eax +; CHECK-NEXT: retq + %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 0, i32 8) + ret i32 %res +} + +define i32 @test_x86_avx512_ucomi_sd_eq_sae(<2 x double> %a0, <2 x double> %a1) { +; CHECK-LABEL: test_x86_avx512_ucomi_sd_eq_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vucomisd {sae}, %xmm1, %xmm0 +; CHECK-NEXT: sete %al +; CHECK-NEXT: movzbl %al, %eax +; CHECK-NEXT: retq + %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 8, i32 8) + ret i32 %res +} + +define i32 @test_x86_avx512_comi_sd_eq(<2 x double> %a0, <2 x double> %a1) { +; CHECK-LABEL: test_x86_avx512_comi_sd_eq: +; CHECK: ## BB#0: +; CHECK-NEXT: vcomisd %xmm1, %xmm0 +; CHECK-NEXT: sete %al +; CHECK-NEXT: movzbl %al, %eax +; CHECK-NEXT: retq + %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 0, i32 4) + ret i32 %res +} + +define i32 @test_x86_avx512_ucomi_sd_eq(<2 x double> %a0, <2 x double> %a1) { +; CHECK-LABEL: test_x86_avx512_ucomi_sd_eq: +; CHECK: ## BB#0: +; CHECK-NEXT: vucomisd %xmm1, %xmm0 +; CHECK-NEXT: sete %al +; CHECK-NEXT: movzbl %al, %eax +; CHECK-NEXT: retq + %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 8, i32 4) + ret i32 %res +} + +define i32 @test_x86_avx512_comi_sd_lt_sae(<2 x double> %a0, <2 x double> %a1) { +; CHECK-LABEL: test_x86_avx512_comi_sd_lt_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vcomisd {sae}, %xmm1, %xmm0 +; CHECK-NEXT: sbbl %eax, %eax +; CHECK-NEXT: andl $1, %eax +; CHECK-NEXT: retq + %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 1, i32 8) + ret i32 %res +} + +define i32 @test_x86_avx512_ucomi_sd_lt_sae(<2 x double> %a0, <2 x double> %a1) { +; CHECK-LABEL: test_x86_avx512_ucomi_sd_lt_sae: +; CHECK: ## BB#0: +; CHECK-NEXT: vucomisd {sae}, %xmm1, %xmm0 +; CHECK-NEXT: sbbl %eax, %eax +; CHECK-NEXT: andl $1, %eax +; CHECK-NEXT: retq + %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 9, i32 8) + ret i32 %res +} + +define i32 @test_x86_avx512_comi_sd_lt(<2 x double> %a0, <2 x double> %a1) { +; CHECK-LABEL: test_x86_avx512_comi_sd_lt: +; CHECK: ## BB#0: +; CHECK-NEXT: vcomisd %xmm1, %xmm0 +; CHECK-NEXT: sbbl %eax, %eax +; CHECK-NEXT: andl $1, %eax +; CHECK-NEXT: retq + %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 1, i32 4) + ret i32 %res +} + +define i32 @test_x86_avx512_ucomi_sd_lt(<2 x double> %a0, <2 x double> %a1) { +; CHECK-LABEL: test_x86_avx512_ucomi_sd_lt: +; CHECK: ## BB#0: +; CHECK-NEXT: vucomisd %xmm1, %xmm0 +; CHECK-NEXT: sbbl %eax, %eax +; CHECK-NEXT: andl $1, %eax +; CHECK-NEXT: retq + %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 9, i32 4) + ret i32 %res +} + +declare i32 @llvm.x86.avx512.vcomi.sd(<2 x double>, <2 x double>, i32, i32) + +define i32 @test_x86_avx512_ucomi_ss_lt(<4 x float> %a0, <4 x float> %a1) { +; CHECK-LABEL: test_x86_avx512_ucomi_ss_lt: +; CHECK: ## BB#0: +; CHECK-NEXT: vucomiss %xmm1, %xmm0 +; CHECK-NEXT: sbbl %eax, %eax +; CHECK-NEXT: andl $1, %eax +; CHECK-NEXT: retq + %res = call i32 @llvm.x86.avx512.vcomi.ss(<4 x float> %a0, <4 x float> %a1, i32 9, i32 4) + ret i32 %res +} + +declare i32 @llvm.x86.avx512.vcomi.ss(<4 x float>, <4 x float>, i32, i32) +declare <4 x float> @llvm.x86.avx512.mask.move.ss(<4 x float>, <4 x float>, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_move_ss_rrk(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_move_ss_rrk: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovss %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.move.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) + ret <4 x float> %res +} + +define <4 x float>@test_int_x86_avx512_mask_move_ss_rrkz(<4 x float> %x0, <4 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_move_ss_rrkz: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1} {z} +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.move.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> zeroinitializer, i8 %x2) + ret <4 x float> %res +} + +define <4 x float>@test_int_x86_avx512_mask_move_ss_rr(<4 x float> %x0, <4 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_move_ss_rr: +; CHECK: ## BB#0: +; CHECK-NEXT: vmovss %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.move.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> zeroinitializer, i8 -1) + ret <4 x float> %res +} + +declare <2 x double> @llvm.x86.avx512.mask.move.sd(<2 x double>, <2 x double>, <2 x double>, i8) +define <2 x double>@test_int_x86_avx512_mask_move_sd_rr(<2 x double> %x0, <2 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_move_sd_rr: +; CHECK: ## BB#0: +; CHECK-NEXT: vmovsd %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.move.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> zeroinitializer, i8 -1) + ret <2 x double> %res +} + +define <2 x double>@test_int_x86_avx512_mask_move_sd_rrkz(<2 x double> %x0, <2 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_move_sd_rrkz: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovsd %xmm1, %xmm0, %xmm0 {%k1} {z} +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.move.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> zeroinitializer, i8 %x2) + ret <2 x double> %res +} + +define <2 x double>@test_int_x86_avx512_mask_move_sd_rrk(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_move_sd_rrk: +; CHECK: ## BB#0: +; CHECK-NEXT: andl $1, %edi +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vmovsd %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vmovaps %zmm2, %zmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.move.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) + ret <2 x double> %res +} + +declare <16 x float> @llvm.x86.avx512.mask.broadcastf32x4.512(<4 x float>, <16 x float>, i16) + +define <16 x float>@test_int_x86_avx512_mask_broadcastf32x4_512(<4 x float> %x0, <16 x float> %x2, i16 %mask) { +; CHECK-LABEL: test_int_x86_avx512_mask_broadcastf32x4_512: +; CHECK: kmovw %edi, %k1 +; CHECK: vshuff32x4 $0, %zmm0, %zmm0, %zmm2 {%k1} {z} +; CHECK: vshuff32x4 $0, %zmm0, %zmm0, %zmm1 {%k1} +; CHECK: vshuff32x4 $0, %zmm0, %zmm0, %zmm0 +; CHECK: vaddps %zmm1, %zmm0, %zmm0 +; CHECK: vaddps %zmm0, %zmm2, %zmm0 + + %res1 = call <16 x float> @llvm.x86.avx512.mask.broadcastf32x4.512(<4 x float> %x0, <16 x float> %x2, i16 -1) + %res2 = call <16 x float> @llvm.x86.avx512.mask.broadcastf32x4.512(<4 x float> %x0, <16 x float> %x2, i16 %mask) + %res3 = call <16 x float> @llvm.x86.avx512.mask.broadcastf32x4.512(<4 x float> %x0, <16 x float> zeroinitializer, i16 %mask) + %res4 = fadd <16 x float> %res1, %res2 + %res5 = fadd <16 x float> %res3, %res4 + ret <16 x float> %res5 +} + +declare <8 x double> @llvm.x86.avx512.mask.broadcastf64x4.512(<4 x double>, <8 x double>, i8) + +define <8 x double>@test_int_x86_avx512_mask_broadcastf64x4_512(<4 x double> %x0, <8 x double> %x2, i8 %mask) { +; CHECK-LABEL: test_int_x86_avx512_mask_broadcastf64x4_512: +; CHECK: kmovw %eax, %k1 +; CHECK: vshuff64x2 $68, %zmm0, %zmm0, %zmm2 {%k1} {z} +; CHECK: vshuff64x2 $68, %zmm0, %zmm0, %zmm1 {%k1} +; CHECK: vshuff64x2 $68, %zmm0, %zmm0, %zmm0 +; CHECK: vaddpd %zmm1, %zmm0, %zmm0 +; CHECK: vaddpd %zmm0, %zmm2, %zmm0 + + %res1 = call <8 x double> @llvm.x86.avx512.mask.broadcastf64x4.512(<4 x double> %x0, <8 x double> %x2, i8 -1) + %res2 = call <8 x double> @llvm.x86.avx512.mask.broadcastf64x4.512(<4 x double> %x0, <8 x double> %x2, i8 %mask) + %res3 = call <8 x double> @llvm.x86.avx512.mask.broadcastf64x4.512(<4 x double> %x0, <8 x double> zeroinitializer, i8 %mask) + %res4 = fadd <8 x double> %res1, %res2 + %res5 = fadd <8 x double> %res3, %res4 + ret <8 x double> %res5 +} + +declare <16 x i32> @llvm.x86.avx512.mask.broadcasti32x4.512(<4 x i32>, <16 x i32>, i16) + +define <16 x i32>@test_int_x86_avx512_mask_broadcasti32x4_512(<4 x i32> %x0, <16 x i32> %x2, i16 %mask) { +; CHECK-LABEL: test_int_x86_avx512_mask_broadcasti32x4_512: +; CHECK: kmovw %edi, %k1 +; CHECK: vshufi32x4 $0, %zmm0, %zmm0, %zmm2 {%k1} {z} +; CHECK: vshufi32x4 $0, %zmm0, %zmm0, %zmm1 {%k1} +; CHECK: vshufi32x4 $0, %zmm0, %zmm0, %zmm0 +; CHECK: vpaddd %zmm1, %zmm0, %zmm0 +; CHECK: vpaddd %zmm0, %zmm2, %zmm0 + + %res1 = call <16 x i32> @llvm.x86.avx512.mask.broadcasti32x4.512(<4 x i32> %x0, <16 x i32> %x2, i16 -1) + %res2 = call <16 x i32> @llvm.x86.avx512.mask.broadcasti32x4.512(<4 x i32> %x0, <16 x i32> %x2, i16 %mask) + %res3 = call <16 x i32> @llvm.x86.avx512.mask.broadcasti32x4.512(<4 x i32> %x0, <16 x i32> zeroinitializer, i16 %mask) + %res4 = add <16 x i32> %res1, %res2 + %res5 = add <16 x i32> %res3, %res4 + ret <16 x i32> %res5 +} + +declare <8 x i64> @llvm.x86.avx512.mask.broadcasti64x4.512(<4 x i64>, <8 x i64>, i8) + +define <8 x i64>@test_int_x86_avx512_mask_broadcasti64x4_512(<4 x i64> %x0, <8 x i64> %x2, i8 %mask) { +; CHECK-LABEL: test_int_x86_avx512_mask_broadcasti64x4_512: +; CHECK: kmovw %eax, %k1 +; CHECK: vshufi64x2 $68, %zmm0, %zmm0, %zmm2 {%k1} {z} +; CHECK: vshufi64x2 $68, %zmm0, %zmm0, %zmm1 {%k1} +; CHECK: vshufi64x2 $68, %zmm0, %zmm0, %zmm0 +; CHECK: vpaddq %zmm1, %zmm0, %zmm0 +; CHECK: vpaddq %zmm0, %zmm2, %zmm0 + + %res1 = call <8 x i64> @llvm.x86.avx512.mask.broadcasti64x4.512(<4 x i64> %x0, <8 x i64> %x2, i8 -1) + %res2 = call <8 x i64> @llvm.x86.avx512.mask.broadcasti64x4.512(<4 x i64> %x0, <8 x i64> %x2, i8 %mask) + %res3 = call <8 x i64> @llvm.x86.avx512.mask.broadcasti64x4.512(<4 x i64> %x0, <8 x i64> zeroinitializer, i8 %mask) + %res4 = add <8 x i64> %res1, %res2 + %res5 = add <8 x i64> %res3, %res4 + ret <8 x i64> %res5 +} + diff --git a/test/CodeGen/X86/avx512-logic.ll b/test/CodeGen/X86/avx512-logic.ll index 140ce3b1ec56..c973b706e8fc 100644 --- a/test/CodeGen/X86/avx512-logic.ll +++ b/test/CodeGen/X86/avx512-logic.ll @@ -1,9 +1,14 @@ -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s --check-prefix=ALL --check-prefix=KNL +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s --check-prefix=ALL --check-prefix=SKX + -; CHECK-LABEL: vpandd -; CHECK: vpandd %zmm -; CHECK: ret define <16 x i32> @vpandd(<16 x i32> %a, <16 x i32> %b) nounwind uwtable readnone ssp { +; ALL-LABEL: vpandd: +; ALL: ## BB#0: ## %entry +; ALL-NEXT: vpaddd {{.*}}(%rip){1to16}, %zmm0, %zmm0 +; ALL-NEXT: vpandd %zmm1, %zmm0, %zmm0 +; ALL-NEXT: retq entry: ; Force the execution domain with an add. %a2 = add <16 x i32> %a, %x } -; CHECK-LABEL: vpord -; CHECK: vpord %zmm -; CHECK: ret define <16 x i32> @vpord(<16 x i32> %a, <16 x i32> %b) nounwind uwtable readnone ssp { +; ALL-LABEL: vpord: +; ALL: ## BB#0: ## %entry +; ALL-NEXT: vpaddd {{.*}}(%rip){1to16}, %zmm0, %zmm0 +; ALL-NEXT: vpord %zmm1, %zmm0, %zmm0 +; ALL-NEXT: retq entry: ; Force the execution domain with an add. %a2 = add <16 x i32> %a, %x } -; CHECK-LABEL: vpxord -; CHECK: vpxord %zmm -; CHECK: ret define <16 x i32> @vpxord(<16 x i32> %a, <16 x i32> %b) nounwind uwtable readnone ssp { +; ALL-LABEL: vpxord: +; ALL: ## BB#0: ## %entry +; ALL-NEXT: vpaddd {{.*}}(%rip){1to16}, %zmm0, %zmm0 +; ALL-NEXT: vpxord %zmm1, %zmm0, %zmm0 +; ALL-NEXT: retq entry: ; Force the execution domain with an add. %a2 = add <16 x i32> %a, %x } -; CHECK-LABEL: vpandq -; CHECK: vpandq %zmm -; CHECK: ret define <8 x i64> @vpandq(<8 x i64> %a, <8 x i64> %b) nounwind uwtable readnone ssp { +; ALL-LABEL: vpandq: +; ALL: ## BB#0: ## %entry +; ALL-NEXT: vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0 +; ALL-NEXT: vpandq %zmm1, %zmm0, %zmm0 +; ALL-NEXT: retq entry: ; Force the execution domain with an add. %a2 = add <8 x i64> %a, @@ -47,10 +58,12 @@ entry: ret <8 x i64> %x } -; CHECK-LABEL: vporq -; CHECK: vporq %zmm -; CHECK: ret define <8 x i64> @vporq(<8 x i64> %a, <8 x i64> %b) nounwind uwtable readnone ssp { +; ALL-LABEL: vporq: +; ALL: ## BB#0: ## %entry +; ALL-NEXT: vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0 +; ALL-NEXT: vporq %zmm1, %zmm0, %zmm0 +; ALL-NEXT: retq entry: ; Force the execution domain with an add. %a2 = add <8 x i64> %a, @@ -58,10 +71,12 @@ entry: ret <8 x i64> %x } -; CHECK-LABEL: vpxorq -; CHECK: vpxorq %zmm -; CHECK: ret define <8 x i64> @vpxorq(<8 x i64> %a, <8 x i64> %b) nounwind uwtable readnone ssp { +; ALL-LABEL: vpxorq: +; ALL: ## BB#0: ## %entry +; ALL-NEXT: vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0 +; ALL-NEXT: vpxorq %zmm1, %zmm0, %zmm0 +; ALL-NEXT: retq entry: ; Force the execution domain with an add. %a2 = add <8 x i64> %a, @@ -70,28 +85,31 @@ entry: } -; CHECK-LABEL: orq_broadcast -; CHECK: vporq LCP{{.*}}(%rip){1to8}, %zmm0, %zmm0 -; CHECK: ret define <8 x i64> @orq_broadcast(<8 x i64> %a) nounwind { +; ALL-LABEL: orq_broadcast: +; ALL: ## BB#0: +; ALL-NEXT: vporq {{.*}}(%rip){1to8}, %zmm0, %zmm0 +; ALL-NEXT: retq %b = or <8 x i64> %a, ret <8 x i64> %b } -; CHECK-LABEL: andd512fold -; CHECK: vpandd (% -; CHECK: ret define <16 x i32> @andd512fold(<16 x i32> %y, <16 x i32>* %x) { +; ALL-LABEL: andd512fold: +; ALL: ## BB#0: ## %entry +; ALL-NEXT: vpandd (%rdi), %zmm0, %zmm0 +; ALL-NEXT: retq entry: %a = load <16 x i32>, <16 x i32>* %x, align 4 %b = and <16 x i32> %y, %a ret <16 x i32> %b } -; CHECK-LABEL: andqbrst -; CHECK: vpandq (%rdi){1to8}, %zmm -; CHECK: ret define <8 x i64> @andqbrst(<8 x i64> %p1, i64* %ap) { +; ALL-LABEL: andqbrst: +; ALL: ## BB#0: ## %entry +; ALL-NEXT: vpandq (%rdi){1to8}, %zmm0, %zmm0 +; ALL-NEXT: retq entry: %a = load i64, i64* %ap, align 8 %b = insertelement <8 x i64> undef, i64 %a, i32 0 @@ -99,3 +117,93 @@ entry: %d = and <8 x i64> %p1, %c ret <8 x i64>%d } + +define <64 x i8> @and_v64i8(<64 x i8> %a, <64 x i8> %b) { +; KNL-LABEL: and_v64i8: +; KNL: ## BB#0: +; KNL-NEXT: vandps %ymm2, %ymm0, %ymm0 +; KNL-NEXT: vandps %ymm3, %ymm1, %ymm1 +; KNL-NEXT: retq +; +; SKX-LABEL: and_v64i8: +; SKX: ## BB#0: +; SKX-NEXT: vpandq %zmm1, %zmm0, %zmm0 +; SKX-NEXT: retq + %res = and <64 x i8> %a, %b + ret <64 x i8> %res +} + +define <64 x i8> @or_v64i8(<64 x i8> %a, <64 x i8> %b) { +; KNL-LABEL: or_v64i8: +; KNL: ## BB#0: +; KNL-NEXT: vorps %ymm2, %ymm0, %ymm0 +; KNL-NEXT: vorps %ymm3, %ymm1, %ymm1 +; KNL-NEXT: retq +; +; SKX-LABEL: or_v64i8: +; SKX: ## BB#0: +; SKX-NEXT: vporq %zmm1, %zmm0, %zmm0 +; SKX-NEXT: retq + %res = or <64 x i8> %a, %b + ret <64 x i8> %res +} + +define <64 x i8> @xor_v64i8(<64 x i8> %a, <64 x i8> %b) { +; KNL-LABEL: xor_v64i8: +; KNL: ## BB#0: +; KNL-NEXT: vxorps %ymm2, %ymm0, %ymm0 +; KNL-NEXT: vxorps %ymm3, %ymm1, %ymm1 +; KNL-NEXT: retq +; +; SKX-LABEL: xor_v64i8: +; SKX: ## BB#0: +; SKX-NEXT: vpxorq %zmm1, %zmm0, %zmm0 +; SKX-NEXT: retq + %res = xor <64 x i8> %a, %b + ret <64 x i8> %res +} + +define <32 x i16> @and_v32i16(<32 x i16> %a, <32 x i16> %b) { +; KNL-LABEL: and_v32i16: +; KNL: ## BB#0: +; KNL-NEXT: vandps %ymm2, %ymm0, %ymm0 +; KNL-NEXT: vandps %ymm3, %ymm1, %ymm1 +; KNL-NEXT: retq +; +; SKX-LABEL: and_v32i16: +; SKX: ## BB#0: +; SKX-NEXT: vpandq %zmm1, %zmm0, %zmm0 +; SKX-NEXT: retq + %res = and <32 x i16> %a, %b + ret <32 x i16> %res +} + +define <32 x i16> @or_v32i16(<32 x i16> %a, <32 x i16> %b) { +; KNL-LABEL: or_v32i16: +; KNL: ## BB#0: +; KNL-NEXT: vorps %ymm2, %ymm0, %ymm0 +; KNL-NEXT: vorps %ymm3, %ymm1, %ymm1 +; KNL-NEXT: retq +; +; SKX-LABEL: or_v32i16: +; SKX: ## BB#0: +; SKX-NEXT: vporq %zmm1, %zmm0, %zmm0 +; SKX-NEXT: retq + %res = or <32 x i16> %a, %b + ret <32 x i16> %res +} + +define <32 x i16> @xor_v32i16(<32 x i16> %a, <32 x i16> %b) { +; KNL-LABEL: xor_v32i16: +; KNL: ## BB#0: +; KNL-NEXT: vxorps %ymm2, %ymm0, %ymm0 +; KNL-NEXT: vxorps %ymm3, %ymm1, %ymm1 +; KNL-NEXT: retq +; +; SKX-LABEL: xor_v32i16: +; SKX: ## BB#0: +; SKX-NEXT: vpxorq %zmm1, %zmm0, %zmm0 +; SKX-NEXT: retq + %res = xor <32 x i16> %a, %b + ret <32 x i16> %res +} diff --git a/test/CodeGen/X86/avx512-mask-op.ll b/test/CodeGen/X86/avx512-mask-op.ll index d2efd7d6db6e..015c70a6ba08 100644 --- a/test/CodeGen/X86/avx512-mask-op.ll +++ b/test/CodeGen/X86/avx512-mask-op.ll @@ -1,39 +1,48 @@ -; RUN: llc < %s -march=x86-64 -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s --check-prefix=KNL --check-prefix=CHECK -; RUN: llc < %s -march=x86-64 -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s --check-prefix=SKX --check-prefix=CHECK +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -march=x86-64 -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s --check-prefix=CHECK --check-prefix=KNL +; RUN: llc < %s -march=x86-64 -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s --check-prefix=CHECK --check-prefix=SKX -; CHECK-LABEL: mask16 -; CHECK: kmovw -; CHECK-NEXT: knotw -; CHECK-NEXT: kmovw define i16 @mask16(i16 %x) { +; CHECK-LABEL: mask16: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k0 +; CHECK-NEXT: knotw %k0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq %m0 = bitcast i16 %x to <16 x i1> %m1 = xor <16 x i1> %m0, %ret = bitcast <16 x i1> %m1 to i16 ret i16 %ret } -; CHECK-LABEL: mask8 -; KNL: kmovw -; KNL-NEXT: knotw -; KNL-NEXT: kmovw -; SKX: kmovb -; SKX-NEXT: knotb -; SKX-NEXT: kmovb - define i8 @mask8(i8 %x) { +; KNL-LABEL: mask8: +; KNL: ## BB#0: +; KNL-NEXT: movzbl %dil, %eax +; KNL-NEXT: kmovw %eax, %k0 +; KNL-NEXT: knotw %k0, %k0 +; KNL-NEXT: kmovw %k0, %eax +; KNL-NEXT: retq +; +; SKX-LABEL: mask8: +; SKX: ## BB#0: +; SKX-NEXT: kmovb %edi, %k0 +; SKX-NEXT: knotb %k0, %k0 +; SKX-NEXT: kmovb %k0, %eax +; SKX-NEXT: retq %m0 = bitcast i8 %x to <8 x i1> %m1 = xor <8 x i1> %m0, %ret = bitcast <8 x i1> %m1 to i8 ret i8 %ret } -; CHECK-LABEL: mask16_mem -; CHECK: kmovw ([[ARG1:%rdi|%rcx]]), %k{{[0-7]}} -; CHECK-NEXT: knotw -; CHECK-NEXT: kmovw %k{{[0-7]}}, ([[ARG1]]) -; CHECK: ret - define void @mask16_mem(i16* %ptr) { +; CHECK-LABEL: mask16_mem: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw (%rdi), %k0 +; CHECK-NEXT: knotw %k0, %k0 +; CHECK-NEXT: kmovw %k0, (%rdi) +; CHECK-NEXT: retq %x = load i16, i16* %ptr, align 4 %m0 = bitcast i16 %x to <16 x i1> %m1 = xor <16 x i1> %m0, @@ -42,15 +51,20 @@ define void @mask16_mem(i16* %ptr) { ret void } -; CHECK-LABEL: mask8_mem -; KNL: kmovw ([[ARG1]]), %k{{[0-7]}} -; KNL-NEXT: knotw -; KNL-NEXT: kmovw %k{{[0-7]}}, ([[ARG1]]) -; SKX: kmovb ([[ARG1]]), %k{{[0-7]}} -; SKX-NEXT: knotb -; SKX-NEXT: kmovb %k{{[0-7]}}, ([[ARG1]]) - define void @mask8_mem(i8* %ptr) { +; KNL-LABEL: mask8_mem: +; KNL: ## BB#0: +; KNL-NEXT: kmovw (%rdi), %k0 +; KNL-NEXT: knotw %k0, %k0 +; KNL-NEXT: kmovw %k0, (%rdi) +; KNL-NEXT: retq +; +; SKX-LABEL: mask8_mem: +; SKX: ## BB#0: +; SKX-NEXT: kmovb (%rdi), %k0 +; SKX-NEXT: knotb %k0, %k0 +; SKX-NEXT: kmovb %k0, (%rdi) +; SKX-NEXT: retq %x = load i8, i8* %ptr, align 4 %m0 = bitcast i8 %x to <8 x i1> %m1 = xor <8 x i1> %m0, @@ -59,11 +73,16 @@ define void @mask8_mem(i8* %ptr) { ret void } -; CHECK-LABEL: mand16 -; CHECK: kandw -; CHECK: kxorw -; CHECK: korw define i16 @mand16(i16 %x, i16 %y) { +; CHECK-LABEL: mand16: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k0 +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: kandw %k1, %k0, %k2 +; CHECK-NEXT: kxorw %k1, %k0, %k0 +; CHECK-NEXT: korw %k0, %k2, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq %ma = bitcast i16 %x to <16 x i1> %mb = bitcast i16 %y to <16 x i1> %mc = and <16 x i1> %ma, %mb @@ -73,56 +92,68 @@ define i16 @mand16(i16 %x, i16 %y) { ret i16 %ret } -; CHECK-LABEL: shuf_test1 -; CHECK: kshiftrw $8 define i8 @shuf_test1(i16 %v) nounwind { +; KNL-LABEL: shuf_test1: +; KNL: ## BB#0: +; KNL-NEXT: kmovw %edi, %k0 +; KNL-NEXT: kshiftrw $8, %k0, %k0 +; KNL-NEXT: kmovw %k0, %eax +; KNL-NEXT: retq +; +; SKX-LABEL: shuf_test1: +; SKX: ## BB#0: +; SKX-NEXT: kmovw %edi, %k0 +; SKX-NEXT: kshiftrw $8, %k0, %k0 +; SKX-NEXT: kmovb %k0, %eax +; SKX-NEXT: retq %v1 = bitcast i16 %v to <16 x i1> %mask = shufflevector <16 x i1> %v1, <16 x i1> undef, <8 x i32> %mask1 = bitcast <8 x i1> %mask to i8 ret i8 %mask1 } -; CHECK-LABEL: zext_test1 -; CHECK: kshiftlw -; CHECK: kshiftrw -; CHECK: kmovw - define i32 @zext_test1(<16 x i32> %a, <16 x i32> %b) { +; CHECK-LABEL: zext_test1: +; CHECK: ## BB#0: +; CHECK-NEXT: vpcmpnleud %zmm1, %zmm0, %k0 +; CHECK-NEXT: kshiftlw $10, %k0, %k0 +; CHECK-NEXT: kshiftrw $15, %k0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: andl $1, %eax +; CHECK-NEXT: retq %cmp_res = icmp ugt <16 x i32> %a, %b %cmp_res.i1 = extractelement <16 x i1> %cmp_res, i32 5 %res = zext i1 %cmp_res.i1 to i32 ret i32 %res -} - -; CHECK-LABEL: zext_test2 -; CHECK: kshiftlw -; CHECK: kshiftrw -; CHECK: kmovw - -define i16 @zext_test2(<16 x i32> %a, <16 x i32> %b) { +}define i16 @zext_test2(<16 x i32> %a, <16 x i32> %b) { %cmp_res = icmp ugt <16 x i32> %a, %b %cmp_res.i1 = extractelement <16 x i1> %cmp_res, i32 5 %res = zext i1 %cmp_res.i1 to i16 ret i16 %res -} - -; CHECK-LABEL: zext_test3 -; CHECK: kshiftlw -; CHECK: kshiftrw -; CHECK: kmovw - -define i8 @zext_test3(<16 x i32> %a, <16 x i32> %b) { +}define i8 @zext_test3(<16 x i32> %a, <16 x i32> %b) { %cmp_res = icmp ugt <16 x i32> %a, %b %cmp_res.i1 = extractelement <16 x i1> %cmp_res, i32 5 %res = zext i1 %cmp_res.i1 to i8 ret i8 %res } -; CHECK-LABEL: conv1 -; KNL: kmovw %k0, %eax -; KNL: movb %al, (%rdi) -; SKX: kmovb %k0, (%rdi) define i8 @conv1(<8 x i1>* %R) { +; KNL-LABEL: conv1: +; KNL: ## BB#0: ## %entry +; KNL-NEXT: kxnorw %k0, %k0, %k0 +; KNL-NEXT: kmovw %k0, %eax +; KNL-NEXT: movb %al, (%rdi) +; KNL-NEXT: movb $-2, -{{[0-9]+}}(%rsp) +; KNL-NEXT: movb $-2, %al +; KNL-NEXT: retq +; +; SKX-LABEL: conv1: +; SKX: ## BB#0: ## %entry +; SKX-NEXT: kxnorw %k0, %k0, %k0 +; SKX-NEXT: kmovb %k0, (%rdi) +; SKX-NEXT: movb $-2, -{{[0-9]+}}(%rsp) +; SKX-NEXT: movb $-2, %al +; SKX-NEXT: retq entry: store <8 x i1> , <8 x i1>* %R @@ -133,12 +164,27 @@ entry: ret i8 %mask_convert } -; SKX-LABEL: test4 -; SKX: vpcmpgt -; SKX: knot -; SKX: vpcmpgt -; SKX: vpmovm2d define <4 x i32> @test4(<4 x i64> %x, <4 x i64> %y, <4 x i64> %x1, <4 x i64> %y1) { +; KNL-LABEL: test4: +; KNL: ## BB#0: +; KNL-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0 +; KNL-NEXT: vpmovqd %zmm0, %ymm0 +; KNL-NEXT: vpslld $31, %xmm0, %xmm0 +; KNL-NEXT: vpsrad $31, %xmm0, %xmm0 +; KNL-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm1 +; KNL-NEXT: vpmovqd %zmm1, %ymm1 +; KNL-NEXT: vpslld $31, %xmm1, %xmm1 +; KNL-NEXT: vpsrad $31, %xmm1, %xmm1 +; KNL-NEXT: vpcmpgtd %xmm1, %xmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test4: +; SKX: ## BB#0: +; SKX-NEXT: vpcmpgtq %ymm3, %ymm2, %k0 +; SKX-NEXT: knotw %k0, %k1 +; SKX-NEXT: vpcmpgtq %ymm1, %ymm0, %k0 {%k1} +; SKX-NEXT: vpmovm2d %k0, %xmm0 +; SKX-NEXT: retq %x_gt_y = icmp sgt <4 x i64> %x, %y %x1_gt_y1 = icmp sgt <4 x i64> %x1, %y1 %res = icmp sgt <4 x i1>%x_gt_y, %x1_gt_y1 @@ -146,30 +192,27 @@ define <4 x i32> @test4(<4 x i64> %x, <4 x i64> %y, <4 x i64> %x1, <4 x i64> %y1 ret <4 x i32> %resse } -; SKX-LABEL: test5 -; SKX: vpcmpgt -; SKX: knot -; SKX: vpcmpgt -; SKX: vpmovm2q define <2 x i64> @test5(<2 x i64> %x, <2 x i64> %y, <2 x i64> %x1, <2 x i64> %y1) { +; KNL-LABEL: test5: +; KNL: ## BB#0: +; KNL-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm0 +; KNL-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm1 +; KNL-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test5: +; SKX: ## BB#0: +; SKX-NEXT: vpcmpgtq %xmm0, %xmm1, %k0 +; SKX-NEXT: knotw %k0, %k1 +; SKX-NEXT: vpcmpgtq %xmm3, %xmm2, %k0 {%k1} +; SKX-NEXT: vpmovm2q %k0, %xmm0 +; SKX-NEXT: retq %x_gt_y = icmp slt <2 x i64> %x, %y %x1_gt_y1 = icmp sgt <2 x i64> %x1, %y1 %res = icmp slt <2 x i1>%x_gt_y, %x1_gt_y1 %resse = sext <2 x i1>%res to <2 x i64> ret <2 x i64> %resse -} - -; KNL-LABEL: test6 -; KNL: vpmovsxbd -; KNL: vpandd -; KNL: kmovw %eax, %k1 -; KNL vptestmd {{.*}}, %k0 {%k1} - -; SKX-LABEL: test6 -; SKX: vpmovb2m -; SKX: kmovw %eax, %k1 -; SKX: kandw -define void @test6(<16 x i1> %mask) { +}define void @test6(<16 x i1> %mask) { allocas: %a= and <16 x i1> %mask, %b = bitcast <16 x i1> %a to i16 @@ -182,19 +225,30 @@ true: false: ret void } - -; KNL-LABEL: test7 -; KNL: vpmovsxwq -; KNL: vpandq -; KNL: vptestmq {{.*}}, %k0 -; KNL: korw - -; SKX-LABEL: test7 -; SKX: vpmovw2m -; SKX: kmovb %eax, %k1 -; SKX: korb - define void @test7(<8 x i1> %mask) { +; KNL-LABEL: test7: +; KNL: ## BB#0: ## %allocas +; KNL-NEXT: vpmovsxwq %xmm0, %zmm0 +; KNL-NEXT: vpsllq $63, %zmm0, %zmm0 +; KNL-NEXT: vptestmq %zmm0, %zmm0, %k0 +; KNL-NEXT: movb $85, %al +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: kmovw %eax, %k1 +; KNL-NEXT: korw %k1, %k0, %k0 +; KNL-NEXT: kmovw %k0, %eax +; KNL-NEXT: testb %al, %al +; KNL-NEXT: retq +; +; SKX-LABEL: test7: +; SKX: ## BB#0: ## %allocas +; SKX-NEXT: vpsllw $15, %xmm0, %xmm0 +; SKX-NEXT: vpmovw2m %xmm0, %k0 +; SKX-NEXT: movb $85, %al +; SKX-NEXT: kmovb %eax, %k1 +; SKX-NEXT: korb %k1, %k0, %k0 +; SKX-NEXT: kmovb %k0, %eax +; SKX-NEXT: testb %al, %al +; SKX-NEXT: retq allocas: %a= or <8 x i1> %mask, %b = bitcast <8 x i1> %a to i8 @@ -207,22 +261,35 @@ true: false: ret void } - -; KNL-LABEL: test8 -; KNL: vpxord %zmm2, %zmm2, %zmm2 -; KNL: jg -; KNL: vpcmpltud %zmm2, %zmm1, %k1 -; KNL: jmp -; KNL: vpcmpgtd %zmm2, %zmm0, %k1 - -; SKX-LABEL: test8 -; SKX: jg -; SKX: vpcmpltud {{.*}}, %k0 -; SKX: vpmovm2b -; SKX: vpcmpgtd {{.*}}, %k0 -; SKX: vpmovm2b - define <16 x i8> @test8(<16 x i32>%a, <16 x i32>%b, i32 %a1, i32 %b1) { +; KNL-LABEL: test8: +; KNL: ## BB#0: +; KNL-NEXT: vpxord %zmm2, %zmm2, %zmm2 +; KNL-NEXT: cmpl %esi, %edi +; KNL-NEXT: jg LBB14_1 +; KNL-NEXT: ## BB#2: +; KNL-NEXT: vpcmpltud %zmm2, %zmm1, %k1 +; KNL-NEXT: jmp LBB14_3 +; KNL-NEXT: LBB14_1: +; KNL-NEXT: vpcmpgtd %zmm2, %zmm0, %k1 +; KNL-NEXT: LBB14_3: +; KNL-NEXT: vpbroadcastd {{.*}}(%rip), %zmm0 {%k1} {z} +; KNL-NEXT: vpmovdb %zmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test8: +; SKX: ## BB#0: +; SKX-NEXT: vpxord %zmm2, %zmm2, %zmm2 +; SKX-NEXT: cmpl %esi, %edi +; SKX-NEXT: jg LBB14_1 +; SKX-NEXT: ## BB#2: +; SKX-NEXT: vpcmpltud %zmm2, %zmm1, %k0 +; SKX-NEXT: vpmovm2b %k0, %xmm0 +; SKX-NEXT: retq +; SKX-NEXT: LBB14_1: +; SKX-NEXT: vpcmpgtd %zmm2, %zmm0, %k0 +; SKX-NEXT: vpmovm2b %k0, %xmm0 +; SKX-NEXT: retq %cond = icmp sgt i32 %a1, %b1 %cmp1 = icmp sgt <16 x i32> %a, zeroinitializer %cmp2 = icmp ult <16 x i32> %b, zeroinitializer @@ -230,91 +297,121 @@ define <16 x i8> @test8(<16 x i32>%a, <16 x i32>%b, i32 %a1, i32 %b1) { %res = sext <16 x i1> %mix to <16 x i8> ret <16 x i8> %res } - -; KNL-LABEL: test9 -; KNL: jg -; KNL: vpmovsxbd %xmm1, %zmm0 -; KNL: jmp -; KNL: vpmovsxbd %xmm0, %zmm0 - -; SKX-LABEL: test9 -; SKX: vpmovb2m %xmm1, %k0 -; SKX: vpmovm2b %k0, %xmm0 -; SKX: retq -; SKX: vpmovb2m %xmm0, %k0 -; SKX: vpmovm2b %k0, %xmm0 - define <16 x i1> @test9(<16 x i1>%a, <16 x i1>%b, i32 %a1, i32 %b1) { +; KNL-LABEL: test9: +; KNL: ## BB#0: +; KNL-NEXT: cmpl %esi, %edi +; KNL-NEXT: jg LBB15_1 +; KNL-NEXT: ## BB#2: +; KNL-NEXT: vpmovsxbd %xmm1, %zmm0 +; KNL-NEXT: jmp LBB15_3 +; KNL-NEXT: LBB15_1: +; KNL-NEXT: vpmovsxbd %xmm0, %zmm0 +; KNL-NEXT: LBB15_3: +; KNL-NEXT: vpslld $31, %zmm0, %zmm0 +; KNL-NEXT: vptestmd %zmm0, %zmm0, %k1 +; KNL-NEXT: vpbroadcastd {{.*}}(%rip), %zmm0 {%k1} {z} +; KNL-NEXT: vpmovdb %zmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test9: +; SKX: ## BB#0: +; SKX-NEXT: cmpl %esi, %edi +; SKX-NEXT: jg LBB15_1 +; SKX-NEXT: ## BB#2: +; SKX-NEXT: vpsllw $7, %xmm1, %xmm0 +; SKX-NEXT: jmp LBB15_3 +; SKX-NEXT: LBB15_1: +; SKX-NEXT: vpsllw $7, %xmm0, %xmm0 +; SKX-NEXT: LBB15_3: +; SKX-NEXT: vpmovb2m %xmm0, %k0 +; SKX-NEXT: vpmovm2b %k0, %xmm0 +; SKX-NEXT: retq %mask = icmp sgt i32 %a1, %b1 %c = select i1 %mask, <16 x i1>%a, <16 x i1>%b ret <16 x i1>%c -} - -; KNL-LABEL: test10 -; KNL: jg -; KNL: vpmovsxwq %xmm1, %zmm0 -; KNL: jmp -; KNL: vpmovsxwq %xmm0, %zmm0 - -; SKX-LABEL: test10 -; SKX: jg -; SKX: vpmovw2m %xmm1, %k0 -; SKX: vpmovm2w %k0, %xmm0 -; SKX: retq -; SKX: vpmovw2m %xmm0, %k0 -; SKX: vpmovm2w %k0, %xmm0 -define <8 x i1> @test10(<8 x i1>%a, <8 x i1>%b, i32 %a1, i32 %b1) { +}define <8 x i1> @test10(<8 x i1>%a, <8 x i1>%b, i32 %a1, i32 %b1) { %mask = icmp sgt i32 %a1, %b1 %c = select i1 %mask, <8 x i1>%a, <8 x i1>%b ret <8 x i1>%c } -; SKX-LABEL: test11 -; SKX: jg -; SKX: vpmovd2m %xmm1, %k0 -; SKX: vpmovm2d %k0, %xmm0 -; SKX: retq -; SKX: vpmovd2m %xmm0, %k0 -; SKX: vpmovm2d %k0, %xmm0 define <4 x i1> @test11(<4 x i1>%a, <4 x i1>%b, i32 %a1, i32 %b1) { +; KNL-LABEL: test11: +; KNL: ## BB#0: +; KNL-NEXT: cmpl %esi, %edi +; KNL-NEXT: jg LBB17_2 +; KNL-NEXT: ## BB#1: +; KNL-NEXT: vmovaps %zmm1, %zmm0 +; KNL-NEXT: LBB17_2: +; KNL-NEXT: retq +; +; SKX-LABEL: test11: +; SKX: ## BB#0: +; SKX-NEXT: cmpl %esi, %edi +; SKX-NEXT: jg LBB17_1 +; SKX-NEXT: ## BB#2: +; SKX-NEXT: vpslld $31, %xmm1, %xmm0 +; SKX-NEXT: jmp LBB17_3 +; SKX-NEXT: LBB17_1: +; SKX-NEXT: vpslld $31, %xmm0, %xmm0 +; SKX-NEXT: LBB17_3: +; SKX-NEXT: vpmovd2m %xmm0, %k0 +; SKX-NEXT: vpmovm2d %k0, %xmm0 +; SKX-NEXT: retq %mask = icmp sgt i32 %a1, %b1 %c = select i1 %mask, <4 x i1>%a, <4 x i1>%b ret <4 x i1>%c } -; KNL-LABEL: test12 -; KNL: movl %edi, %eax define i32 @test12(i32 %x, i32 %y) { +; CHECK-LABEL: test12: +; CHECK: ## BB#0: +; CHECK-NEXT: movl %edi, %eax +; CHECK-NEXT: retq %a = bitcast i16 21845 to <16 x i1> %b = extractelement <16 x i1> %a, i32 0 %c = select i1 %b, i32 %x, i32 %y ret i32 %c } -; KNL-LABEL: test13 -; KNL: movl %esi, %eax define i32 @test13(i32 %x, i32 %y) { +; CHECK-LABEL: test13: +; CHECK: ## BB#0: +; CHECK-NEXT: movl %esi, %eax +; CHECK-NEXT: retq %a = bitcast i16 21845 to <16 x i1> %b = extractelement <16 x i1> %a, i32 3 %c = select i1 %b, i32 %x, i32 %y ret i32 %c -} - -; SKX-LABEL: test14 -; SKX: movb $11, %al -; SKX: kmovb %eax, %k0 -; SKX: vpmovm2d %k0, %xmm0 - -define <4 x i1> @test14() { +}define <4 x i1> @test14() { %a = bitcast i16 21845 to <16 x i1> %b = extractelement <16 x i1> %a, i32 2 %c = insertelement <4 x i1> , i1 %b, i32 1 ret <4 x i1> %c } -; KNL-LABEL: test15 -; KNL: cmovgw define <16 x i1> @test15(i32 %x, i32 %y) { +; KNL-LABEL: test15: +; KNL: ## BB#0: +; KNL-NEXT: cmpl %esi, %edi +; KNL-NEXT: movw $21845, %ax ## imm = 0x5555 +; KNL-NEXT: movw $1, %cx +; KNL-NEXT: cmovgw %ax, %cx +; KNL-NEXT: kmovw %ecx, %k1 +; KNL-NEXT: vpbroadcastd {{.*}}(%rip), %zmm0 {%k1} {z} +; KNL-NEXT: vpmovdb %zmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test15: +; SKX: ## BB#0: +; SKX-NEXT: cmpl %esi, %edi +; SKX-NEXT: movw $21845, %ax ## imm = 0x5555 +; SKX-NEXT: movw $1, %cx +; SKX-NEXT: cmovgw %ax, %cx +; SKX-NEXT: kmovw %ecx, %k0 +; SKX-NEXT: vpmovm2b %k0, %xmm0 +; SKX-NEXT: retq %a = bitcast i16 21845 to <16 x i1> %b = bitcast i16 1 to <16 x i1> %mask = icmp sgt i32 %x, %y @@ -322,27 +419,914 @@ define <16 x i1> @test15(i32 %x, i32 %y) { ret <16 x i1> %c } -; SKX-LABEL: test16 -; SKX: kxnorw %k1, %k1, %k1 -; SKX: kshiftrw $15, %k1, %k1 -; SKX: kshiftlq $5, %k1, %k1 -; SKX: korq %k1, %k0, %k0 -; SKX: vpmovm2b %k0, %zmm0 define <64 x i8> @test16(i64 %x) { +; KNL-LABEL: test16: +; KNL: ## BB#0: +; KNL-NEXT: pushq %rbp +; KNL-NEXT: Ltmp0: +; KNL-NEXT: .cfi_def_cfa_offset 16 +; KNL-NEXT: Ltmp1: +; KNL-NEXT: .cfi_offset %rbp, -16 +; KNL-NEXT: movq %rsp, %rbp +; KNL-NEXT: Ltmp2: +; KNL-NEXT: .cfi_def_cfa_register %rbp +; KNL-NEXT: pushq %r15 +; KNL-NEXT: pushq %r14 +; KNL-NEXT: pushq %r13 +; KNL-NEXT: pushq %r12 +; KNL-NEXT: pushq %rbx +; KNL-NEXT: andq $-32, %rsp +; KNL-NEXT: subq $128, %rsp +; KNL-NEXT: Ltmp3: +; KNL-NEXT: .cfi_offset %rbx, -56 +; KNL-NEXT: Ltmp4: +; KNL-NEXT: .cfi_offset %r12, -48 +; KNL-NEXT: Ltmp5: +; KNL-NEXT: .cfi_offset %r13, -40 +; KNL-NEXT: Ltmp6: +; KNL-NEXT: .cfi_offset %r14, -32 +; KNL-NEXT: Ltmp7: +; KNL-NEXT: .cfi_offset %r15, -24 +; KNL-NEXT: movq %rdi, %rax +; KNL-NEXT: shrq $32, %rax +; KNL-NEXT: movl %eax, {{[0-9]+}}(%rsp) +; KNL-NEXT: movl $271, %eax ## imm = 0x10F +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: movl %edi, %ecx +; KNL-NEXT: andl $1, %ecx +; KNL-NEXT: vmovd %ecx, %xmm0 +; KNL-NEXT: movl $257, %ecx ## imm = 0x101 +; KNL-NEXT: bextrl %ecx, %edi, %ecx +; KNL-NEXT: vpinsrb $1, %ecx, %xmm0, %xmm0 +; KNL-NEXT: movl $258, %ecx ## imm = 0x102 +; KNL-NEXT: bextrl %ecx, %edi, %ecx +; KNL-NEXT: vpinsrb $2, %ecx, %xmm0, %xmm0 +; KNL-NEXT: movl $259, %ecx ## imm = 0x103 +; KNL-NEXT: bextrl %ecx, %edi, %ecx +; KNL-NEXT: vpinsrb $3, %ecx, %xmm0, %xmm0 +; KNL-NEXT: movl $260, %ecx ## imm = 0x104 +; KNL-NEXT: bextrl %ecx, %edi, %ecx +; KNL-NEXT: vpinsrb $4, %ecx, %xmm0, %xmm0 +; KNL-NEXT: movl $261, %ecx ## imm = 0x105 +; KNL-NEXT: bextrl %ecx, %edi, %ecx +; KNL-NEXT: vpinsrb $5, %ecx, %xmm0, %xmm0 +; KNL-NEXT: movl $262, %ecx ## imm = 0x106 +; KNL-NEXT: bextrl %ecx, %edi, %ecx +; KNL-NEXT: vpinsrb $6, %ecx, %xmm0, %xmm0 +; KNL-NEXT: movl $263, %ecx ## imm = 0x107 +; KNL-NEXT: bextrl %ecx, %edi, %ecx +; KNL-NEXT: vpinsrb $7, %ecx, %xmm0, %xmm0 +; KNL-NEXT: movl $264, %ecx ## imm = 0x108 +; KNL-NEXT: bextrl %ecx, %edi, %ecx +; KNL-NEXT: vpinsrb $8, %ecx, %xmm0, %xmm0 +; KNL-NEXT: movl $265, %ecx ## imm = 0x109 +; KNL-NEXT: bextrl %ecx, %edi, %ecx +; KNL-NEXT: vpinsrb $9, %ecx, %xmm0, %xmm0 +; KNL-NEXT: movl $266, %ecx ## imm = 0x10A +; KNL-NEXT: bextrl %ecx, %edi, %ecx +; KNL-NEXT: vpinsrb $10, %ecx, %xmm0, %xmm0 +; KNL-NEXT: movl $267, %ecx ## imm = 0x10B +; KNL-NEXT: bextrl %ecx, %edi, %ecx +; KNL-NEXT: vpinsrb $11, %ecx, %xmm0, %xmm0 +; KNL-NEXT: movl $268, %ecx ## imm = 0x10C +; KNL-NEXT: bextrl %ecx, %edi, %ecx +; KNL-NEXT: vpinsrb $12, %ecx, %xmm0, %xmm0 +; KNL-NEXT: movl $269, %ecx ## imm = 0x10D +; KNL-NEXT: bextrl %ecx, %edi, %ecx +; KNL-NEXT: vpinsrb $13, %ecx, %xmm0, %xmm0 +; KNL-NEXT: movl $270, %ecx ## imm = 0x10E +; KNL-NEXT: bextrl %ecx, %edi, %ecx +; KNL-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0 +; KNL-NEXT: vpinsrb $15, %eax, %xmm0, %xmm1 +; KNL-NEXT: movl $1, %eax +; KNL-NEXT: vpinsrb $5, %eax, %xmm1, %xmm0 +; KNL-NEXT: movl {{[0-9]+}}(%rsp), %r15d +; KNL-NEXT: movq %r15, %rdx +; KNL-NEXT: shrq $17, %rdx +; KNL-NEXT: andb $1, %dl +; KNL-NEXT: je LBB22_2 +; KNL-NEXT: ## BB#1: +; KNL-NEXT: movb $-1, %dl +; KNL-NEXT: LBB22_2: +; KNL-NEXT: movq %r15, %r11 +; KNL-NEXT: shrq $16, %r11 +; KNL-NEXT: andb $1, %r11b +; KNL-NEXT: je LBB22_4 +; KNL-NEXT: ## BB#3: +; KNL-NEXT: movb $-1, %r11b +; KNL-NEXT: LBB22_4: +; KNL-NEXT: movq %r15, %r10 +; KNL-NEXT: shrq $18, %r10 +; KNL-NEXT: andb $1, %r10b +; KNL-NEXT: je LBB22_6 +; KNL-NEXT: ## BB#5: +; KNL-NEXT: movb $-1, %r10b +; KNL-NEXT: LBB22_6: +; KNL-NEXT: movq %r15, %r9 +; KNL-NEXT: shrq $19, %r9 +; KNL-NEXT: andb $1, %r9b +; KNL-NEXT: je LBB22_8 +; KNL-NEXT: ## BB#7: +; KNL-NEXT: movb $-1, %r9b +; KNL-NEXT: LBB22_8: +; KNL-NEXT: movq %r15, %rbx +; KNL-NEXT: shrq $20, %rbx +; KNL-NEXT: andb $1, %bl +; KNL-NEXT: je LBB22_10 +; KNL-NEXT: ## BB#9: +; KNL-NEXT: movb $-1, %bl +; KNL-NEXT: LBB22_10: +; KNL-NEXT: movq %r15, %r12 +; KNL-NEXT: shrq $21, %r12 +; KNL-NEXT: andb $1, %r12b +; KNL-NEXT: je LBB22_12 +; KNL-NEXT: ## BB#11: +; KNL-NEXT: movb $-1, %r12b +; KNL-NEXT: LBB22_12: +; KNL-NEXT: movq %r15, %r14 +; KNL-NEXT: shrq $22, %r14 +; KNL-NEXT: andb $1, %r14b +; KNL-NEXT: je LBB22_14 +; KNL-NEXT: ## BB#13: +; KNL-NEXT: movb $-1, %r14b +; KNL-NEXT: LBB22_14: +; KNL-NEXT: movq %r15, %r8 +; KNL-NEXT: shrq $23, %r8 +; KNL-NEXT: andb $1, %r8b +; KNL-NEXT: je LBB22_16 +; KNL-NEXT: ## BB#15: +; KNL-NEXT: movb $-1, %r8b +; KNL-NEXT: LBB22_16: +; KNL-NEXT: movq %r15, %r13 +; KNL-NEXT: shrq $24, %r13 +; KNL-NEXT: andb $1, %r13b +; KNL-NEXT: je LBB22_18 +; KNL-NEXT: ## BB#17: +; KNL-NEXT: movb $-1, %r13b +; KNL-NEXT: LBB22_18: +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $25, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB22_20 +; KNL-NEXT: ## BB#19: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB22_20: +; KNL-NEXT: movq %rax, {{[0-9]+}}(%rsp) ## 8-byte Spill +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $26, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB22_22 +; KNL-NEXT: ## BB#21: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB22_22: +; KNL-NEXT: movq %rax, {{[0-9]+}}(%rsp) ## 8-byte Spill +; KNL-NEXT: movl $272, %esi ## imm = 0x110 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $27, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB22_24 +; KNL-NEXT: ## BB#23: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB22_24: +; KNL-NEXT: movq %rax, {{[0-9]+}}(%rsp) ## 8-byte Spill +; KNL-NEXT: movl $273, %eax ## imm = 0x111 +; KNL-NEXT: bextrl %esi, %edi, %esi +; KNL-NEXT: movq %r15, %rcx +; KNL-NEXT: shrq $28, %rcx +; KNL-NEXT: andb $1, %cl +; KNL-NEXT: je LBB22_26 +; KNL-NEXT: ## BB#25: +; KNL-NEXT: movb $-1, %cl +; KNL-NEXT: LBB22_26: +; KNL-NEXT: movq %rcx, {{[0-9]+}}(%rsp) ## 8-byte Spill +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: vmovd %esi, %xmm2 +; KNL-NEXT: movl $274, %esi ## imm = 0x112 +; KNL-NEXT: movq %r15, %rcx +; KNL-NEXT: shrq $29, %rcx +; KNL-NEXT: andb $1, %cl +; KNL-NEXT: je LBB22_28 +; KNL-NEXT: ## BB#27: +; KNL-NEXT: movb $-1, %cl +; KNL-NEXT: LBB22_28: +; KNL-NEXT: movq %rcx, {{[0-9]+}}(%rsp) ## 8-byte Spill +; KNL-NEXT: vpinsrb $1, %eax, %xmm2, %xmm2 +; KNL-NEXT: bextrl %esi, %edi, %eax +; KNL-NEXT: movzbl %r11b, %esi +; KNL-NEXT: movq %r15, %rcx +; KNL-NEXT: shrq $30, %rcx +; KNL-NEXT: andb $1, %cl +; KNL-NEXT: je LBB22_30 +; KNL-NEXT: ## BB#29: +; KNL-NEXT: movb $-1, %cl +; KNL-NEXT: LBB22_30: +; KNL-NEXT: vpinsrb $2, %eax, %xmm2, %xmm2 +; KNL-NEXT: movl $275, %eax ## imm = 0x113 +; KNL-NEXT: bextrl %eax, %edi, %r11d +; KNL-NEXT: movzbl %dl, %edx +; KNL-NEXT: vmovd %esi, %xmm3 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $31, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB22_32 +; KNL-NEXT: ## BB#31: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB22_32: +; KNL-NEXT: movq %rax, {{[0-9]+}}(%rsp) ## 8-byte Spill +; KNL-NEXT: movq %rcx, {{[0-9]+}}(%rsp) ## 8-byte Spill +; KNL-NEXT: vpinsrb $3, %r11d, %xmm2, %xmm2 +; KNL-NEXT: movl $276, %eax ## imm = 0x114 +; KNL-NEXT: bextrl %eax, %edi, %esi +; KNL-NEXT: movl $277, %r11d ## imm = 0x115 +; KNL-NEXT: vpinsrb $1, %edx, %xmm3, %xmm3 +; KNL-NEXT: movzbl %r10b, %r10d +; KNL-NEXT: movb %r15b, %al +; KNL-NEXT: shrb %al +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB22_34 +; KNL-NEXT: ## BB#33: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB22_34: +; KNL-NEXT: vpinsrb $4, %esi, %xmm2, %xmm2 +; KNL-NEXT: bextrl %r11d, %edi, %edx +; KNL-NEXT: movl $278, %r11d ## imm = 0x116 +; KNL-NEXT: vpinsrb $2, %r10d, %xmm3, %xmm3 +; KNL-NEXT: movzbl %r9b, %esi +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: movq %r15, %rcx +; KNL-NEXT: shlq $63, %rcx +; KNL-NEXT: sarq $63, %rcx +; KNL-NEXT: vmovd %ecx, %xmm4 +; KNL-NEXT: vpinsrb $1, %eax, %xmm4, %xmm4 +; KNL-NEXT: movb %r15b, %al +; KNL-NEXT: shrb $2, %al +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB22_36 +; KNL-NEXT: ## BB#35: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB22_36: +; KNL-NEXT: vpinsrb $5, %edx, %xmm2, %xmm2 +; KNL-NEXT: bextrl %r11d, %edi, %edx +; KNL-NEXT: movl $279, %r9d ## imm = 0x117 +; KNL-NEXT: vpinsrb $3, %esi, %xmm3, %xmm3 +; KNL-NEXT: movzbl %bl, %ebx +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $2, %eax, %xmm4, %xmm4 +; KNL-NEXT: movb %r15b, %al +; KNL-NEXT: shrb $3, %al +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB22_38 +; KNL-NEXT: ## BB#37: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB22_38: +; KNL-NEXT: vpinsrb $6, %edx, %xmm2, %xmm2 +; KNL-NEXT: bextrl %r9d, %edi, %edx +; KNL-NEXT: movl $280, %esi ## imm = 0x118 +; KNL-NEXT: vpinsrb $4, %ebx, %xmm3, %xmm3 +; KNL-NEXT: movzbl %r12b, %ebx +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $3, %eax, %xmm4, %xmm4 +; KNL-NEXT: movb %r15b, %al +; KNL-NEXT: shrb $4, %al +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB22_40 +; KNL-NEXT: ## BB#39: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB22_40: +; KNL-NEXT: vpinsrb $7, %edx, %xmm2, %xmm2 +; KNL-NEXT: bextrl %esi, %edi, %ecx +; KNL-NEXT: movl $281, %edx ## imm = 0x119 +; KNL-NEXT: vpinsrb $5, %ebx, %xmm3, %xmm3 +; KNL-NEXT: movzbl %r14b, %esi +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $4, %eax, %xmm4, %xmm4 +; KNL-NEXT: movb %r15b, %al +; KNL-NEXT: shrb $5, %al +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB22_42 +; KNL-NEXT: ## BB#41: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB22_42: +; KNL-NEXT: vpinsrb $8, %ecx, %xmm2, %xmm2 +; KNL-NEXT: bextrl %edx, %edi, %ecx +; KNL-NEXT: movl $282, %edx ## imm = 0x11A +; KNL-NEXT: vpinsrb $6, %esi, %xmm3, %xmm3 +; KNL-NEXT: movzbl %r8b, %esi +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $5, %eax, %xmm4, %xmm4 +; KNL-NEXT: movb %r15b, %bl +; KNL-NEXT: shrb $6, %bl +; KNL-NEXT: andb $1, %bl +; KNL-NEXT: je LBB22_44 +; KNL-NEXT: ## BB#43: +; KNL-NEXT: movb $-1, %bl +; KNL-NEXT: LBB22_44: +; KNL-NEXT: vpinsrb $9, %ecx, %xmm2, %xmm2 +; KNL-NEXT: bextrl %edx, %edi, %eax +; KNL-NEXT: movl $283, %ecx ## imm = 0x11B +; KNL-NEXT: vpinsrb $7, %esi, %xmm3, %xmm3 +; KNL-NEXT: movzbl %r13b, %esi +; KNL-NEXT: movzbl %bl, %edx +; KNL-NEXT: vpinsrb $6, %edx, %xmm4, %xmm4 +; KNL-NEXT: movb %r15b, %bl +; KNL-NEXT: shrb $7, %bl +; KNL-NEXT: je LBB22_46 +; KNL-NEXT: ## BB#45: +; KNL-NEXT: movb $-1, %bl +; KNL-NEXT: LBB22_46: +; KNL-NEXT: vpinsrb $10, %eax, %xmm2, %xmm2 +; KNL-NEXT: bextrl %ecx, %edi, %ecx +; KNL-NEXT: movl $284, %edx ## imm = 0x11C +; KNL-NEXT: vpinsrb $8, %esi, %xmm3, %xmm3 +; KNL-NEXT: movq {{[0-9]+}}(%rsp), %rax ## 8-byte Reload +; KNL-NEXT: movzbl %al, %esi +; KNL-NEXT: movzbl %bl, %eax +; KNL-NEXT: vpinsrb $7, %eax, %xmm4, %xmm4 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $8, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB22_48 +; KNL-NEXT: ## BB#47: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB22_48: +; KNL-NEXT: vpinsrb $11, %ecx, %xmm2, %xmm2 +; KNL-NEXT: bextrl %edx, %edi, %ecx +; KNL-NEXT: movl $285, %edx ## imm = 0x11D +; KNL-NEXT: vpinsrb $9, %esi, %xmm3, %xmm3 +; KNL-NEXT: movq {{[0-9]+}}(%rsp), %rsi ## 8-byte Reload +; KNL-NEXT: movzbl %sil, %esi +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $8, %eax, %xmm4, %xmm4 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $9, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB22_50 +; KNL-NEXT: ## BB#49: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB22_50: +; KNL-NEXT: vpinsrb $12, %ecx, %xmm2, %xmm2 +; KNL-NEXT: bextrl %edx, %edi, %ecx +; KNL-NEXT: movl $286, %edx ## imm = 0x11E +; KNL-NEXT: vpinsrb $10, %esi, %xmm3, %xmm3 +; KNL-NEXT: movq {{[0-9]+}}(%rsp), %rsi ## 8-byte Reload +; KNL-NEXT: movzbl %sil, %esi +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $9, %eax, %xmm4, %xmm4 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $10, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB22_52 +; KNL-NEXT: ## BB#51: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB22_52: +; KNL-NEXT: vpinsrb $13, %ecx, %xmm2, %xmm2 +; KNL-NEXT: bextrl %edx, %edi, %edx +; KNL-NEXT: vpinsrb $11, %esi, %xmm3, %xmm3 +; KNL-NEXT: movq {{[0-9]+}}(%rsp), %rcx ## 8-byte Reload +; KNL-NEXT: movzbl %cl, %ecx +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $10, %eax, %xmm4, %xmm4 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $11, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB22_54 +; KNL-NEXT: ## BB#53: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB22_54: +; KNL-NEXT: vpinsrb $14, %edx, %xmm2, %xmm2 +; KNL-NEXT: shrl $31, %edi +; KNL-NEXT: vpinsrb $12, %ecx, %xmm3, %xmm3 +; KNL-NEXT: movq {{[0-9]+}}(%rsp), %rcx ## 8-byte Reload +; KNL-NEXT: movzbl %cl, %ecx +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $11, %eax, %xmm4, %xmm4 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $12, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB22_56 +; KNL-NEXT: ## BB#55: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB22_56: +; KNL-NEXT: vpinsrb $15, %edi, %xmm2, %xmm2 +; KNL-NEXT: vpinsrb $13, %ecx, %xmm3, %xmm3 +; KNL-NEXT: movq {{[0-9]+}}(%rsp), %rcx ## 8-byte Reload +; KNL-NEXT: movzbl %cl, %ecx +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $12, %eax, %xmm4, %xmm4 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $13, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB22_58 +; KNL-NEXT: ## BB#57: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB22_58: +; KNL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; KNL-NEXT: vpinsrb $14, %ecx, %xmm3, %xmm2 +; KNL-NEXT: movq {{[0-9]+}}(%rsp), %rcx ## 8-byte Reload +; KNL-NEXT: movzbl %cl, %ecx +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $13, %eax, %xmm4, %xmm3 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $14, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB22_60 +; KNL-NEXT: ## BB#59: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB22_60: +; KNL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; KNL-NEXT: vpinsrb $15, %ecx, %xmm2, %xmm1 +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $14, %eax, %xmm3, %xmm2 +; KNL-NEXT: shrq $15, %r15 +; KNL-NEXT: andb $1, %r15b +; KNL-NEXT: je LBB22_62 +; KNL-NEXT: ## BB#61: +; KNL-NEXT: movb $-1, %r15b +; KNL-NEXT: LBB22_62: +; KNL-NEXT: movzbl %r15b, %eax +; KNL-NEXT: vpinsrb $15, %eax, %xmm2, %xmm2 +; KNL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1 +; KNL-NEXT: vpsllw $7, %ymm0, %ymm0 +; KNL-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; KNL-NEXT: vpxor %ymm2, %ymm2, %ymm2 +; KNL-NEXT: vpcmpgtb %ymm0, %ymm2, %ymm0 +; KNL-NEXT: leaq -40(%rbp), %rsp +; KNL-NEXT: popq %rbx +; KNL-NEXT: popq %r12 +; KNL-NEXT: popq %r13 +; KNL-NEXT: popq %r14 +; KNL-NEXT: popq %r15 +; KNL-NEXT: popq %rbp +; KNL-NEXT: retq +; +; SKX-LABEL: test16: +; SKX: ## BB#0: +; SKX-NEXT: kmovq %rdi, %k0 +; SKX-NEXT: kxnorw %k0, %k0, %k1 +; SKX-NEXT: kshiftrw $15, %k1, %k1 +; SKX-NEXT: kshiftlq $5, %k1, %k1 +; SKX-NEXT: korq %k1, %k0, %k0 +; SKX-NEXT: vpmovm2b %k0, %zmm0 +; SKX-NEXT: retq %a = bitcast i64 %x to <64 x i1> %b = insertelement <64 x i1>%a, i1 true, i32 5 %c = sext <64 x i1>%b to <64 x i8> ret <64 x i8>%c } -; SKX-LABEL: test17 -; SKX: setg %al -; SKX: andl $1, %eax -; SKX: kmovw %eax, %k1 -; SKX: kshiftlq $5, %k1, %k1 -; SKX: korq %k1, %k0, %k0 -; SKX: vpmovm2b %k0, %zmm0 define <64 x i8> @test17(i64 %x, i32 %y, i32 %z) { +; KNL-LABEL: test17: +; KNL: ## BB#0: +; KNL-NEXT: pushq %rbp +; KNL-NEXT: Ltmp8: +; KNL-NEXT: .cfi_def_cfa_offset 16 +; KNL-NEXT: Ltmp9: +; KNL-NEXT: .cfi_offset %rbp, -16 +; KNL-NEXT: movq %rsp, %rbp +; KNL-NEXT: Ltmp10: +; KNL-NEXT: .cfi_def_cfa_register %rbp +; KNL-NEXT: pushq %r15 +; KNL-NEXT: pushq %r14 +; KNL-NEXT: pushq %r13 +; KNL-NEXT: pushq %r12 +; KNL-NEXT: pushq %rbx +; KNL-NEXT: andq $-32, %rsp +; KNL-NEXT: subq $128, %rsp +; KNL-NEXT: Ltmp11: +; KNL-NEXT: .cfi_offset %rbx, -56 +; KNL-NEXT: Ltmp12: +; KNL-NEXT: .cfi_offset %r12, -48 +; KNL-NEXT: Ltmp13: +; KNL-NEXT: .cfi_offset %r13, -40 +; KNL-NEXT: Ltmp14: +; KNL-NEXT: .cfi_offset %r14, -32 +; KNL-NEXT: Ltmp15: +; KNL-NEXT: .cfi_offset %r15, -24 +; KNL-NEXT: movq %rdi, %rax +; KNL-NEXT: shrq $32, %rax +; KNL-NEXT: movl %eax, {{[0-9]+}}(%rsp) +; KNL-NEXT: movl %edi, %eax +; KNL-NEXT: andl $1, %eax +; KNL-NEXT: vmovd %eax, %xmm0 +; KNL-NEXT: movl $257, %eax ## imm = 0x101 +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: vpinsrb $1, %eax, %xmm0, %xmm0 +; KNL-NEXT: movl $258, %eax ## imm = 0x102 +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: vpinsrb $2, %eax, %xmm0, %xmm0 +; KNL-NEXT: movl $259, %eax ## imm = 0x103 +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: vpinsrb $3, %eax, %xmm0, %xmm0 +; KNL-NEXT: movl $260, %eax ## imm = 0x104 +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: vpinsrb $4, %eax, %xmm0, %xmm0 +; KNL-NEXT: movl $261, %eax ## imm = 0x105 +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: vpinsrb $5, %eax, %xmm0, %xmm0 +; KNL-NEXT: movl $262, %eax ## imm = 0x106 +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: vpinsrb $6, %eax, %xmm0, %xmm0 +; KNL-NEXT: movl $263, %eax ## imm = 0x107 +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: vpinsrb $7, %eax, %xmm0, %xmm0 +; KNL-NEXT: movl $264, %eax ## imm = 0x108 +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: vpinsrb $8, %eax, %xmm0, %xmm0 +; KNL-NEXT: movl $265, %eax ## imm = 0x109 +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: vpinsrb $9, %eax, %xmm0, %xmm0 +; KNL-NEXT: movl $266, %eax ## imm = 0x10A +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: vpinsrb $10, %eax, %xmm0, %xmm0 +; KNL-NEXT: movl $267, %eax ## imm = 0x10B +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: vpinsrb $11, %eax, %xmm0, %xmm0 +; KNL-NEXT: movl $268, %eax ## imm = 0x10C +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: vpinsrb $12, %eax, %xmm0, %xmm0 +; KNL-NEXT: movl $269, %eax ## imm = 0x10D +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: vpinsrb $13, %eax, %xmm0, %xmm0 +; KNL-NEXT: movl $270, %eax ## imm = 0x10E +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: vpinsrb $14, %eax, %xmm0, %xmm0 +; KNL-NEXT: movl $271, %eax ## imm = 0x10F +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: vpinsrb $15, %eax, %xmm0, %xmm1 +; KNL-NEXT: cmpl %edx, %esi +; KNL-NEXT: setg %al +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $5, %eax, %xmm1, %xmm0 +; KNL-NEXT: movl {{[0-9]+}}(%rsp), %r15d +; KNL-NEXT: movq %r15, %rdx +; KNL-NEXT: shrq $17, %rdx +; KNL-NEXT: andb $1, %dl +; KNL-NEXT: je LBB23_2 +; KNL-NEXT: ## BB#1: +; KNL-NEXT: movb $-1, %dl +; KNL-NEXT: LBB23_2: +; KNL-NEXT: movq %r15, %r11 +; KNL-NEXT: shrq $16, %r11 +; KNL-NEXT: andb $1, %r11b +; KNL-NEXT: je LBB23_4 +; KNL-NEXT: ## BB#3: +; KNL-NEXT: movb $-1, %r11b +; KNL-NEXT: LBB23_4: +; KNL-NEXT: movq %r15, %r10 +; KNL-NEXT: shrq $18, %r10 +; KNL-NEXT: andb $1, %r10b +; KNL-NEXT: je LBB23_6 +; KNL-NEXT: ## BB#5: +; KNL-NEXT: movb $-1, %r10b +; KNL-NEXT: LBB23_6: +; KNL-NEXT: movq %r15, %r9 +; KNL-NEXT: shrq $19, %r9 +; KNL-NEXT: andb $1, %r9b +; KNL-NEXT: je LBB23_8 +; KNL-NEXT: ## BB#7: +; KNL-NEXT: movb $-1, %r9b +; KNL-NEXT: LBB23_8: +; KNL-NEXT: movq %r15, %rbx +; KNL-NEXT: shrq $20, %rbx +; KNL-NEXT: andb $1, %bl +; KNL-NEXT: je LBB23_10 +; KNL-NEXT: ## BB#9: +; KNL-NEXT: movb $-1, %bl +; KNL-NEXT: LBB23_10: +; KNL-NEXT: movq %r15, %r12 +; KNL-NEXT: shrq $21, %r12 +; KNL-NEXT: andb $1, %r12b +; KNL-NEXT: je LBB23_12 +; KNL-NEXT: ## BB#11: +; KNL-NEXT: movb $-1, %r12b +; KNL-NEXT: LBB23_12: +; KNL-NEXT: movq %r15, %r14 +; KNL-NEXT: shrq $22, %r14 +; KNL-NEXT: andb $1, %r14b +; KNL-NEXT: je LBB23_14 +; KNL-NEXT: ## BB#13: +; KNL-NEXT: movb $-1, %r14b +; KNL-NEXT: LBB23_14: +; KNL-NEXT: movq %r15, %r8 +; KNL-NEXT: shrq $23, %r8 +; KNL-NEXT: andb $1, %r8b +; KNL-NEXT: je LBB23_16 +; KNL-NEXT: ## BB#15: +; KNL-NEXT: movb $-1, %r8b +; KNL-NEXT: LBB23_16: +; KNL-NEXT: movq %r15, %r13 +; KNL-NEXT: shrq $24, %r13 +; KNL-NEXT: andb $1, %r13b +; KNL-NEXT: je LBB23_18 +; KNL-NEXT: ## BB#17: +; KNL-NEXT: movb $-1, %r13b +; KNL-NEXT: LBB23_18: +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $25, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB23_20 +; KNL-NEXT: ## BB#19: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB23_20: +; KNL-NEXT: movq %rax, {{[0-9]+}}(%rsp) ## 8-byte Spill +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $26, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB23_22 +; KNL-NEXT: ## BB#21: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB23_22: +; KNL-NEXT: movq %rax, {{[0-9]+}}(%rsp) ## 8-byte Spill +; KNL-NEXT: movl $272, %esi ## imm = 0x110 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $27, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB23_24 +; KNL-NEXT: ## BB#23: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB23_24: +; KNL-NEXT: movq %rax, {{[0-9]+}}(%rsp) ## 8-byte Spill +; KNL-NEXT: movl $273, %eax ## imm = 0x111 +; KNL-NEXT: bextrl %esi, %edi, %esi +; KNL-NEXT: movq %r15, %rcx +; KNL-NEXT: shrq $28, %rcx +; KNL-NEXT: andb $1, %cl +; KNL-NEXT: je LBB23_26 +; KNL-NEXT: ## BB#25: +; KNL-NEXT: movb $-1, %cl +; KNL-NEXT: LBB23_26: +; KNL-NEXT: movq %rcx, {{[0-9]+}}(%rsp) ## 8-byte Spill +; KNL-NEXT: bextrl %eax, %edi, %eax +; KNL-NEXT: vmovd %esi, %xmm2 +; KNL-NEXT: movl $274, %esi ## imm = 0x112 +; KNL-NEXT: movq %r15, %rcx +; KNL-NEXT: shrq $29, %rcx +; KNL-NEXT: andb $1, %cl +; KNL-NEXT: je LBB23_28 +; KNL-NEXT: ## BB#27: +; KNL-NEXT: movb $-1, %cl +; KNL-NEXT: LBB23_28: +; KNL-NEXT: movq %rcx, {{[0-9]+}}(%rsp) ## 8-byte Spill +; KNL-NEXT: vpinsrb $1, %eax, %xmm2, %xmm2 +; KNL-NEXT: bextrl %esi, %edi, %eax +; KNL-NEXT: movzbl %r11b, %esi +; KNL-NEXT: movq %r15, %rcx +; KNL-NEXT: shrq $30, %rcx +; KNL-NEXT: andb $1, %cl +; KNL-NEXT: je LBB23_30 +; KNL-NEXT: ## BB#29: +; KNL-NEXT: movb $-1, %cl +; KNL-NEXT: LBB23_30: +; KNL-NEXT: vpinsrb $2, %eax, %xmm2, %xmm2 +; KNL-NEXT: movl $275, %eax ## imm = 0x113 +; KNL-NEXT: bextrl %eax, %edi, %r11d +; KNL-NEXT: movzbl %dl, %edx +; KNL-NEXT: vmovd %esi, %xmm3 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $31, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB23_32 +; KNL-NEXT: ## BB#31: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB23_32: +; KNL-NEXT: movq %rax, {{[0-9]+}}(%rsp) ## 8-byte Spill +; KNL-NEXT: movq %rcx, {{[0-9]+}}(%rsp) ## 8-byte Spill +; KNL-NEXT: vpinsrb $3, %r11d, %xmm2, %xmm2 +; KNL-NEXT: movl $276, %eax ## imm = 0x114 +; KNL-NEXT: bextrl %eax, %edi, %esi +; KNL-NEXT: movl $277, %r11d ## imm = 0x115 +; KNL-NEXT: vpinsrb $1, %edx, %xmm3, %xmm3 +; KNL-NEXT: movzbl %r10b, %r10d +; KNL-NEXT: movb %r15b, %al +; KNL-NEXT: shrb %al +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB23_34 +; KNL-NEXT: ## BB#33: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB23_34: +; KNL-NEXT: vpinsrb $4, %esi, %xmm2, %xmm2 +; KNL-NEXT: bextrl %r11d, %edi, %edx +; KNL-NEXT: movl $278, %r11d ## imm = 0x116 +; KNL-NEXT: vpinsrb $2, %r10d, %xmm3, %xmm3 +; KNL-NEXT: movzbl %r9b, %esi +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: movq %r15, %rcx +; KNL-NEXT: shlq $63, %rcx +; KNL-NEXT: sarq $63, %rcx +; KNL-NEXT: vmovd %ecx, %xmm4 +; KNL-NEXT: vpinsrb $1, %eax, %xmm4, %xmm4 +; KNL-NEXT: movb %r15b, %al +; KNL-NEXT: shrb $2, %al +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB23_36 +; KNL-NEXT: ## BB#35: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB23_36: +; KNL-NEXT: vpinsrb $5, %edx, %xmm2, %xmm2 +; KNL-NEXT: bextrl %r11d, %edi, %edx +; KNL-NEXT: movl $279, %r9d ## imm = 0x117 +; KNL-NEXT: vpinsrb $3, %esi, %xmm3, %xmm3 +; KNL-NEXT: movzbl %bl, %ebx +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $2, %eax, %xmm4, %xmm4 +; KNL-NEXT: movb %r15b, %al +; KNL-NEXT: shrb $3, %al +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB23_38 +; KNL-NEXT: ## BB#37: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB23_38: +; KNL-NEXT: vpinsrb $6, %edx, %xmm2, %xmm2 +; KNL-NEXT: bextrl %r9d, %edi, %edx +; KNL-NEXT: movl $280, %esi ## imm = 0x118 +; KNL-NEXT: vpinsrb $4, %ebx, %xmm3, %xmm3 +; KNL-NEXT: movzbl %r12b, %ebx +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $3, %eax, %xmm4, %xmm4 +; KNL-NEXT: movb %r15b, %al +; KNL-NEXT: shrb $4, %al +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB23_40 +; KNL-NEXT: ## BB#39: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB23_40: +; KNL-NEXT: vpinsrb $7, %edx, %xmm2, %xmm2 +; KNL-NEXT: bextrl %esi, %edi, %ecx +; KNL-NEXT: movl $281, %edx ## imm = 0x119 +; KNL-NEXT: vpinsrb $5, %ebx, %xmm3, %xmm3 +; KNL-NEXT: movzbl %r14b, %esi +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $4, %eax, %xmm4, %xmm4 +; KNL-NEXT: movb %r15b, %al +; KNL-NEXT: shrb $5, %al +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB23_42 +; KNL-NEXT: ## BB#41: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB23_42: +; KNL-NEXT: vpinsrb $8, %ecx, %xmm2, %xmm2 +; KNL-NEXT: bextrl %edx, %edi, %ecx +; KNL-NEXT: movl $282, %edx ## imm = 0x11A +; KNL-NEXT: vpinsrb $6, %esi, %xmm3, %xmm3 +; KNL-NEXT: movzbl %r8b, %esi +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $5, %eax, %xmm4, %xmm4 +; KNL-NEXT: movb %r15b, %bl +; KNL-NEXT: shrb $6, %bl +; KNL-NEXT: andb $1, %bl +; KNL-NEXT: je LBB23_44 +; KNL-NEXT: ## BB#43: +; KNL-NEXT: movb $-1, %bl +; KNL-NEXT: LBB23_44: +; KNL-NEXT: vpinsrb $9, %ecx, %xmm2, %xmm2 +; KNL-NEXT: bextrl %edx, %edi, %eax +; KNL-NEXT: movl $283, %ecx ## imm = 0x11B +; KNL-NEXT: vpinsrb $7, %esi, %xmm3, %xmm3 +; KNL-NEXT: movzbl %r13b, %esi +; KNL-NEXT: movzbl %bl, %edx +; KNL-NEXT: vpinsrb $6, %edx, %xmm4, %xmm4 +; KNL-NEXT: movb %r15b, %bl +; KNL-NEXT: shrb $7, %bl +; KNL-NEXT: je LBB23_46 +; KNL-NEXT: ## BB#45: +; KNL-NEXT: movb $-1, %bl +; KNL-NEXT: LBB23_46: +; KNL-NEXT: vpinsrb $10, %eax, %xmm2, %xmm2 +; KNL-NEXT: bextrl %ecx, %edi, %ecx +; KNL-NEXT: movl $284, %edx ## imm = 0x11C +; KNL-NEXT: vpinsrb $8, %esi, %xmm3, %xmm3 +; KNL-NEXT: movq {{[0-9]+}}(%rsp), %rax ## 8-byte Reload +; KNL-NEXT: movzbl %al, %esi +; KNL-NEXT: movzbl %bl, %eax +; KNL-NEXT: vpinsrb $7, %eax, %xmm4, %xmm4 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $8, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB23_48 +; KNL-NEXT: ## BB#47: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB23_48: +; KNL-NEXT: vpinsrb $11, %ecx, %xmm2, %xmm2 +; KNL-NEXT: bextrl %edx, %edi, %ecx +; KNL-NEXT: movl $285, %edx ## imm = 0x11D +; KNL-NEXT: vpinsrb $9, %esi, %xmm3, %xmm3 +; KNL-NEXT: movq {{[0-9]+}}(%rsp), %rsi ## 8-byte Reload +; KNL-NEXT: movzbl %sil, %esi +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $8, %eax, %xmm4, %xmm4 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $9, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB23_50 +; KNL-NEXT: ## BB#49: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB23_50: +; KNL-NEXT: vpinsrb $12, %ecx, %xmm2, %xmm2 +; KNL-NEXT: bextrl %edx, %edi, %ecx +; KNL-NEXT: movl $286, %edx ## imm = 0x11E +; KNL-NEXT: vpinsrb $10, %esi, %xmm3, %xmm3 +; KNL-NEXT: movq {{[0-9]+}}(%rsp), %rsi ## 8-byte Reload +; KNL-NEXT: movzbl %sil, %esi +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $9, %eax, %xmm4, %xmm4 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $10, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB23_52 +; KNL-NEXT: ## BB#51: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB23_52: +; KNL-NEXT: vpinsrb $13, %ecx, %xmm2, %xmm2 +; KNL-NEXT: bextrl %edx, %edi, %edx +; KNL-NEXT: vpinsrb $11, %esi, %xmm3, %xmm3 +; KNL-NEXT: movq {{[0-9]+}}(%rsp), %rcx ## 8-byte Reload +; KNL-NEXT: movzbl %cl, %ecx +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $10, %eax, %xmm4, %xmm4 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $11, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB23_54 +; KNL-NEXT: ## BB#53: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB23_54: +; KNL-NEXT: vpinsrb $14, %edx, %xmm2, %xmm2 +; KNL-NEXT: shrl $31, %edi +; KNL-NEXT: vpinsrb $12, %ecx, %xmm3, %xmm3 +; KNL-NEXT: movq {{[0-9]+}}(%rsp), %rcx ## 8-byte Reload +; KNL-NEXT: movzbl %cl, %ecx +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $11, %eax, %xmm4, %xmm4 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $12, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB23_56 +; KNL-NEXT: ## BB#55: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB23_56: +; KNL-NEXT: vpinsrb $15, %edi, %xmm2, %xmm2 +; KNL-NEXT: vpinsrb $13, %ecx, %xmm3, %xmm3 +; KNL-NEXT: movq {{[0-9]+}}(%rsp), %rcx ## 8-byte Reload +; KNL-NEXT: movzbl %cl, %ecx +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $12, %eax, %xmm4, %xmm4 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $13, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB23_58 +; KNL-NEXT: ## BB#57: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB23_58: +; KNL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; KNL-NEXT: vpinsrb $14, %ecx, %xmm3, %xmm2 +; KNL-NEXT: movq {{[0-9]+}}(%rsp), %rcx ## 8-byte Reload +; KNL-NEXT: movzbl %cl, %ecx +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $13, %eax, %xmm4, %xmm3 +; KNL-NEXT: movq %r15, %rax +; KNL-NEXT: shrq $14, %rax +; KNL-NEXT: andb $1, %al +; KNL-NEXT: je LBB23_60 +; KNL-NEXT: ## BB#59: +; KNL-NEXT: movb $-1, %al +; KNL-NEXT: LBB23_60: +; KNL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; KNL-NEXT: vpinsrb $15, %ecx, %xmm2, %xmm1 +; KNL-NEXT: movzbl %al, %eax +; KNL-NEXT: vpinsrb $14, %eax, %xmm3, %xmm2 +; KNL-NEXT: shrq $15, %r15 +; KNL-NEXT: andb $1, %r15b +; KNL-NEXT: je LBB23_62 +; KNL-NEXT: ## BB#61: +; KNL-NEXT: movb $-1, %r15b +; KNL-NEXT: LBB23_62: +; KNL-NEXT: movzbl %r15b, %eax +; KNL-NEXT: vpinsrb $15, %eax, %xmm2, %xmm2 +; KNL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1 +; KNL-NEXT: vpsllw $7, %ymm0, %ymm0 +; KNL-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; KNL-NEXT: vpxor %ymm2, %ymm2, %ymm2 +; KNL-NEXT: vpcmpgtb %ymm0, %ymm2, %ymm0 +; KNL-NEXT: leaq -40(%rbp), %rsp +; KNL-NEXT: popq %rbx +; KNL-NEXT: popq %r12 +; KNL-NEXT: popq %r13 +; KNL-NEXT: popq %r14 +; KNL-NEXT: popq %r15 +; KNL-NEXT: popq %rbp +; KNL-NEXT: retq +; +; SKX-LABEL: test17: +; SKX: ## BB#0: +; SKX-NEXT: kmovq %rdi, %k0 +; SKX-NEXT: cmpl %edx, %esi +; SKX-NEXT: setg %al +; SKX-NEXT: andl $1, %eax +; SKX-NEXT: kmovw %eax, %k1 +; SKX-NEXT: kshiftlq $5, %k1, %k1 +; SKX-NEXT: korq %k1, %k0, %k0 +; SKX-NEXT: vpmovm2b %k0, %zmm0 +; SKX-NEXT: retq %a = bitcast i64 %x to <64 x i1> %b = icmp sgt i32 %y, %z %c = insertelement <64 x i1>%a, i1 %b, i32 5 @@ -350,8 +1334,38 @@ define <64 x i8> @test17(i64 %x, i32 %y, i32 %z) { ret <64 x i8>%d } -; KNL-LABEL: test18 define <8 x i1> @test18(i8 %a, i16 %y) { +; KNL-LABEL: test18: +; KNL: ## BB#0: +; KNL-NEXT: movzbl %dil, %eax +; KNL-NEXT: kmovw %eax, %k0 +; KNL-NEXT: kmovw %esi, %k1 +; KNL-NEXT: kshiftlw $7, %k1, %k2 +; KNL-NEXT: kshiftrw $15, %k2, %k2 +; KNL-NEXT: kshiftlw $6, %k1, %k1 +; KNL-NEXT: kshiftrw $15, %k1, %k1 +; KNL-NEXT: kshiftlw $6, %k1, %k1 +; KNL-NEXT: korw %k1, %k0, %k0 +; KNL-NEXT: kshiftlw $7, %k2, %k1 +; KNL-NEXT: korw %k1, %k0, %k1 +; KNL-NEXT: vpbroadcastq {{.*}}(%rip), %zmm0 {%k1} {z} +; KNL-NEXT: vpmovqw %zmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: test18: +; SKX: ## BB#0: +; SKX-NEXT: kmovb %edi, %k0 +; SKX-NEXT: kmovw %esi, %k1 +; SKX-NEXT: kshiftlw $6, %k1, %k2 +; SKX-NEXT: kshiftrw $15, %k2, %k2 +; SKX-NEXT: kshiftlw $7, %k1, %k1 +; SKX-NEXT: kshiftrw $15, %k1, %k1 +; SKX-NEXT: kshiftlb $7, %k1, %k1 +; SKX-NEXT: kshiftlb $6, %k2, %k2 +; SKX-NEXT: korb %k2, %k0, %k0 +; SKX-NEXT: korb %k1, %k0, %k0 +; SKX-NEXT: vpmovm2w %k0, %xmm0 +; SKX-NEXT: retq %b = bitcast i8 %a to <8 x i1> %b1 = bitcast i16 %y to <16 x i1> %el1 = extractelement <16 x i1>%b1, i32 8 @@ -360,50 +1374,76 @@ define <8 x i1> @test18(i8 %a, i16 %y) { %d = insertelement <8 x i1>%c, i1 %el2, i32 6 ret <8 x i1>%d } - -; KNL-LABEL: test19 -; KNL: movzbl %dil, %eax -; KNL: kmovw %eax, %k0 -; KNL: kshiftlw $13, %k0, %k0 -; KNL: kshiftrw $15, %k0, %k0 -; KNL: kmovw %k0, %eax -; KNL: andl $1, %eax -; KNL: testb %al, %al - -define <8 x i1> @test19(i8 %a) { - %b = bitcast i8 %a to <8 x i1> - %c = shufflevector < 8 x i1>%b, <8 x i1>undef, <8 x i32> - ret <8 x i1> %c +define <32 x i16> @test21(<32 x i16> %x , <32 x i1> %mask) nounwind readnone { +; KNL-LABEL: test21: +; KNL: ## BB#0: +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero +; KNL-NEXT: vpsllw $15, %ymm3, %ymm3 +; KNL-NEXT: vpsraw $15, %ymm3, %ymm3 +; KNL-NEXT: vpand %ymm0, %ymm3, %ymm0 +; KNL-NEXT: vextracti128 $1, %ymm2, %xmm2 +; KNL-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero +; KNL-NEXT: vpsllw $15, %ymm2, %ymm2 +; KNL-NEXT: vpsraw $15, %ymm2, %ymm2 +; KNL-NEXT: vpand %ymm1, %ymm2, %ymm1 +; KNL-NEXT: retq +; +; SKX-LABEL: test21: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %ymm1, %ymm1 +; SKX-NEXT: vpmovb2m %ymm1, %k1 +; SKX-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; SKX-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1} +; SKX-NEXT: vmovaps %zmm1, %zmm0 +; SKX-NEXT: retq + %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer + ret <32 x i16> %ret } -; KNL-LABEL: test20 -; KNL: movzbl %dil, %eax -; KNL: kmovw %eax, %k0 -; KNL: kshiftlw $13, %k0, %k1 -; KNL: kshiftrw $15, %k1, %k1 -; KNL: kshiftlw $12, %k0, %k0 -; KNL: kshiftrw $15, %k0, %k0 -; KNL: kshiftlw $4, %k0, %k0 -; KNL: kshiftlw $1, %k1, %k2 -; KNL: korw %k0, %k2, %k0 -; KNL: kshiftlw $6, %k1, %k1 -; KNL: korw %k1, %k0, %k1 -define <8 x i1> @test20(i8 %a, i16 %y) { - %b = bitcast i8 %a to <8 x i1> - %c = shufflevector < 8 x i1>%b, <8 x i1>undef, <8 x i32> - ret <8 x i1> %c +define void @test22(<4 x i1> %a, <4 x i1>* %addr) { +; KNL-LABEL: test22: +; KNL: ## BB#0: +; KNL-NEXT: vpextrd $3, %xmm0, %eax +; KNL-NEXT: andl $1, %eax +; KNL-NEXT: movb %al, (%rdi) +; KNL-NEXT: vpextrd $2, %xmm0, %eax +; KNL-NEXT: andl $1, %eax +; KNL-NEXT: movb %al, (%rdi) +; KNL-NEXT: vpextrd $1, %xmm0, %eax +; KNL-NEXT: andl $1, %eax +; KNL-NEXT: movb %al, (%rdi) +; KNL-NEXT: vmovd %xmm0, %eax +; KNL-NEXT: andl $1, %eax +; KNL-NEXT: movb %al, (%rdi) +; KNL-NEXT: retq +; +; SKX-LABEL: test22: +; SKX: ## BB#0: +; SKX-NEXT: vpslld $31, %xmm0, %xmm0 +; SKX-NEXT: vpmovd2m %xmm0, %k0 +; SKX-NEXT: kmovb %k0, (%rdi) +; SKX-NEXT: retq + store <4 x i1> %a, <4 x i1>* %addr + ret void } -; KNL-LABEL: test21 -; KNL: vpand %ymm -; KNL: vextracti128 $1, %ymm2 -; KNL: vpand %ymm - -; SKX-LABEL: test21 -; SKX: vpmovb2m -; SKX: vmovdqu16 {{.*}}%k1 - -define <32 x i16> @test21(<32 x i16> %x , <32 x i1> %mask) nounwind readnone { - %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer - ret <32 x i16> %ret +define void @test23(<2 x i1> %a, <2 x i1>* %addr) { +; KNL-LABEL: test23: +; KNL: ## BB#0: +; KNL-NEXT: vpextrq $1, %xmm0, %rax +; KNL-NEXT: andl $1, %eax +; KNL-NEXT: movb %al, (%rdi) +; KNL-NEXT: vmovq %xmm0, %rax +; KNL-NEXT: andl $1, %eax +; KNL-NEXT: movb %al, (%rdi) +; KNL-NEXT: retq +; +; SKX-LABEL: test23: +; SKX: ## BB#0: +; SKX-NEXT: vpsllq $63, %xmm0, %xmm0 +; SKX-NEXT: vpmovq2m %xmm0, %k0 +; SKX-NEXT: kmovb %k0, (%rdi) +; SKX-NEXT: retq + store <2 x i1> %a, <2 x i1>* %addr + ret void } diff --git a/test/CodeGen/X86/avx512-skx-insert-subvec.ll b/test/CodeGen/X86/avx512-skx-insert-subvec.ll new file mode 100644 index 000000000000..c54010cd91b9 --- /dev/null +++ b/test/CodeGen/X86/avx512-skx-insert-subvec.ll @@ -0,0 +1,135 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw -mattr=+avx512dq -mattr=+avx512vl| FileCheck %s + +define <8 x i1> @test(<2 x i1> %a) { +; CHECK-LABEL: test: +; CHECK: # BB#0: +; CHECK-NEXT: vpsllq $63, %xmm0, %xmm0 +; CHECK-NEXT: vpmovq2m %xmm0, %k0 +; CHECK-NEXT: kshiftlb $2, %k0, %k0 +; CHECK-NEXT: vpmovm2w %k0, %xmm0 +; CHECK-NEXT: retq + %res = shufflevector <2 x i1> %a, <2 x i1> undef, <8 x i32> + ret <8 x i1> %res +} + +define <8 x i1> @test1(<2 x i1> %a) { +; CHECK-LABEL: test1: +; CHECK: # BB#0: +; CHECK-NEXT: vpsllq $63, %xmm0, %xmm0 +; CHECK-NEXT: vpmovq2m %xmm0, %k0 +; CHECK-NEXT: kshiftlb $4, %k0, %k0 +; CHECK-NEXT: vpmovm2w %k0, %xmm0 +; CHECK-NEXT: retq + %res = shufflevector <2 x i1> %a, <2 x i1> undef, <8 x i32> + ret <8 x i1> %res +} + +define <8 x i1> @test2(<2 x i1> %a) { +; CHECK-LABEL: test2: +; CHECK: # BB#0: +; CHECK-NEXT: vpsllq $63, %xmm0, %xmm0 +; CHECK-NEXT: vpmovq2m %xmm0, %k0 +; CHECK-NEXT: vpmovm2q %k0, %zmm0 +; CHECK-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; CHECK-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,0,1],zmm0[0,1,0,1] +; CHECK-NEXT: vpsllq $63, %zmm0, %zmm0 +; CHECK-NEXT: vpmovq2m %zmm0, %k0 +; CHECK-NEXT: vpmovm2w %k0, %xmm0 +; CHECK-NEXT: retq + %res = shufflevector <2 x i1> %a, <2 x i1> zeroinitializer, <8 x i32> + ret <8 x i1> %res +} + +define <8 x i1> @test3(<4 x i1> %a) { +; CHECK-LABEL: test3: +; CHECK: # BB#0: +; CHECK-NEXT: vpslld $31, %xmm0, %xmm0 +; CHECK-NEXT: vpmovd2m %xmm0, %k0 +; CHECK-NEXT: kshiftlb $4, %k0, %k0 +; CHECK-NEXT: kshiftrb $4, %k0, %k0 +; CHECK-NEXT: vpmovm2w %k0, %xmm0 +; CHECK-NEXT: retq + + %res = shufflevector <4 x i1> %a, <4 x i1> zeroinitializer, <8 x i32> + ret <8 x i1> %res +} + +define <8 x i1> @test4(<4 x i1> %a, <4 x i1>%b) { +; CHECK-LABEL: test4: +; CHECK: # BB#0: +; CHECK-NEXT: vpslld $31, %xmm0, %xmm0 +; CHECK-NEXT: vpmovd2m %xmm0, %k0 +; CHECK-NEXT: kshiftlb $4, %k0, %k0 +; CHECK-NEXT: kshiftrb $4, %k0, %k1 +; CHECK-NEXT: korb %k0, %k1, %k0 +; CHECK-NEXT: vpmovm2w %k0, %xmm0 +; CHECK-NEXT: retq + + %res = shufflevector <4 x i1> %a, <4 x i1> %b, <8 x i32> + ret <8 x i1> %res +} + +define <4 x i1> @test5(<2 x i1> %a, <2 x i1>%b) { +; CHECK-LABEL: test5: +; CHECK: # BB#0: +; CHECK-NEXT: vpsllq $63, %xmm0, %xmm0 +; CHECK-NEXT: vpmovq2m %xmm0, %k0 +; CHECK-NEXT: kshiftlw $2, %k0, %k0 +; CHECK-NEXT: kshiftrw $2, %k0, %k1 +; CHECK-NEXT: korw %k0, %k1, %k0 +; CHECK-NEXT: vpmovm2d %k0, %xmm0 +; CHECK-NEXT: retq + + %res = shufflevector <2 x i1> %a, <2 x i1> %b, <4 x i32> + ret <4 x i1> %res +} + +define <16 x i1> @test6(<2 x i1> %a, <2 x i1>%b) { +; CHECK-LABEL: test6: +; CHECK: # BB#0: +; CHECK-NEXT: vpsllq $63, %xmm0, %xmm0 +; CHECK-NEXT: vpmovq2m %xmm0, %k0 +; CHECK-NEXT: kshiftlw $2, %k0, %k0 +; CHECK-NEXT: kshiftrw $2, %k0, %k1 +; CHECK-NEXT: korw %k0, %k1, %k0 +; CHECK-NEXT: kunpckbw %k0, %k0, %k0 +; CHECK-NEXT: vpmovm2b %k0, %xmm0 +; CHECK-NEXT: retq + + %res = shufflevector <2 x i1> %a, <2 x i1> %b, <16 x i32> + ret <16 x i1> %res +} + +define <32 x i1> @test7(<4 x i1> %a, <4 x i1>%b) { +; CHECK-LABEL: test7: +; CHECK: # BB#0: +; CHECK-NEXT: vpslld $31, %xmm0, %xmm0 +; CHECK-NEXT: vpmovd2m %xmm0, %k0 +; CHECK-NEXT: kshiftlb $4, %k0, %k0 +; CHECK-NEXT: kshiftrb $4, %k0, %k1 +; CHECK-NEXT: korb %k0, %k1, %k0 +; CHECK-NEXT: kunpckbw %k0, %k0, %k0 +; CHECK-NEXT: kunpckwd %k0, %k0, %k0 +; CHECK-NEXT: vpmovm2b %k0, %ymm0 +; CHECK-NEXT: retq + + %res = shufflevector <4 x i1> %a, <4 x i1> %b, <32 x i32> + ret <32 x i1> %res +} + +define <64 x i1> @test8(<8 x i1> %a, <8 x i1>%b) { +; CHECK-LABEL: test8: +; CHECK: # BB#0: +; CHECK-NEXT: vpsllw $15, %xmm1, %xmm1 +; CHECK-NEXT: vpmovw2m %xmm1, %k0 +; CHECK-NEXT: vpsllw $15, %xmm0, %xmm0 +; CHECK-NEXT: vpmovw2m %xmm0, %k1 +; CHECK-NEXT: kunpckdq %k1, %k0, %k0 +; CHECK-NEXT: vpmovm2b %k0, %zmm0 +; CHECK-NEXT: retq + + %res = shufflevector <8 x i1> %a, <8 x i1> %b, <64 x i32> + ret <64 x i1> %res +} + diff --git a/test/CodeGen/X86/avx512-trunc-ext.ll b/test/CodeGen/X86/avx512-trunc-ext.ll deleted file mode 100644 index f25458972e42..000000000000 --- a/test/CodeGen/X86/avx512-trunc-ext.ll +++ /dev/null @@ -1,961 +0,0 @@ -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s --check-prefix=KNL -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s --check-prefix=SKX - - -; KNL-LABEL: trunc_16x32_to_16x8 -; KNL: vpmovdb -; KNL: ret -define <16 x i8> @trunc_16x32_to_16x8(<16 x i32> %i) nounwind readnone { - %x = trunc <16 x i32> %i to <16 x i8> - ret <16 x i8> %x -} - -; KNL-LABEL: trunc_8x64_to_8x16 -; KNL: vpmovqw -; KNL: ret -define <8 x i16> @trunc_8x64_to_8x16(<8 x i64> %i) nounwind readnone { - %x = trunc <8 x i64> %i to <8 x i16> - ret <8 x i16> %x -} - -;SKX-LABEL: zext_8x8mem_to_8x16: -;SKX: ## BB#0: -;SKX-NEXT: vpmovw2m %xmm0, %k1 -;SKX-NEXT: vpmovzxbw (%rdi), %xmm0 {%k1} {z} -;SKX-NEXT: retq -define <8 x i16> @zext_8x8mem_to_8x16(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone { - %a = load <8 x i8>,<8 x i8> *%i,align 1 - %x = zext <8 x i8> %a to <8 x i16> - %ret = select <8 x i1> %mask, <8 x i16> %x, <8 x i16> zeroinitializer - ret <8 x i16> %ret -} - -;SKX-LABEL: sext_8x8mem_to_8x16: -;SKX: ## BB#0: -;SKX-NEXT: vpmovw2m %xmm0, %k1 -;SKX-NEXT: vpmovsxbw (%rdi), %xmm0 {%k1} {z} -;SKX-NEXT: retq -define <8 x i16> @sext_8x8mem_to_8x16(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone { - %a = load <8 x i8>,<8 x i8> *%i,align 1 - %x = sext <8 x i8> %a to <8 x i16> - %ret = select <8 x i1> %mask, <8 x i16> %x, <8 x i16> zeroinitializer - ret <8 x i16> %ret -} - -;SKX-LABEL: zext_16x8mem_to_16x16: -;SKX: ## BB#0: -;SKX-NEXT: vpmovb2m %xmm0, %k1 -;SKX-NEXT: vpmovzxbw (%rdi), %ymm0 {%k1} {z} -;SKX-NEXT: retq -define <16 x i16> @zext_16x8mem_to_16x16(<16 x i8> *%i , <16 x i1> %mask) nounwind readnone { - %a = load <16 x i8>,<16 x i8> *%i,align 1 - %x = zext <16 x i8> %a to <16 x i16> - %ret = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> zeroinitializer - ret <16 x i16> %ret -} - -;SKX-LABEL: sext_16x8mem_to_16x16: -;SKX: ## BB#0: -;SKX-NEXT: vpmovb2m %xmm0, %k1 -;SKX-NEXT: vpmovsxbw (%rdi), %ymm0 {%k1} {z} -;SKX-NEXT: retq -define <16 x i16> @sext_16x8mem_to_16x16(<16 x i8> *%i , <16 x i1> %mask) nounwind readnone { - %a = load <16 x i8>,<16 x i8> *%i,align 1 - %x = sext <16 x i8> %a to <16 x i16> - %ret = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> zeroinitializer - ret <16 x i16> %ret -} - -;SKX-LABEL: zext_16x8_to_16x16: -;SKX: ## BB#0: -;SKX-NEXT: vpmovzxbw %xmm0, %ymm0 -;SKX-NEXT: retq -define <16 x i16> @zext_16x8_to_16x16(<16 x i8> %a ) nounwind readnone { - %x = zext <16 x i8> %a to <16 x i16> - ret <16 x i16> %x -} - -;SKX-LABEL: zext_16x8_to_16x16_mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovb2m %xmm1, %k1 -;SKX-NEXT: vpmovzxbw %xmm0, %ymm0 {%k1} {z} -;SKX-NEXT: retq -define <16 x i16> @zext_16x8_to_16x16_mask(<16 x i8> %a ,<16 x i1> %mask) nounwind readnone { - %x = zext <16 x i8> %a to <16 x i16> - %ret = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> zeroinitializer - ret <16 x i16> %ret -} - -;SKX-LABEL: sext_16x8_to_16x16: -;SKX: ## BB#0: -;SKX-NEXT: vpmovsxbw %xmm0, %ymm0 -;SKX-NEXT: retq -define <16 x i16> @sext_16x8_to_16x16(<16 x i8> %a ) nounwind readnone { - %x = sext <16 x i8> %a to <16 x i16> - ret <16 x i16> %x -} - -;SKX-LABEL: sext_16x8_to_16x16_mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovb2m %xmm1, %k1 -;SKX-NEXT: vpmovsxbw %xmm0, %ymm0 {%k1} {z} -;SKX-NEXT: retq -define <16 x i16> @sext_16x8_to_16x16_mask(<16 x i8> %a ,<16 x i1> %mask) nounwind readnone { - %x = sext <16 x i8> %a to <16 x i16> - %ret = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> zeroinitializer - ret <16 x i16> %ret -} - -;SKX-LABEL: zext_32x8mem_to_32x16: -;SKX: ## BB#0: -;SKX-NEXT: vpmovb2m %ymm0, %k1 -;SKX-NEXT: vpmovzxbw (%rdi), %zmm0 {%k1} {z} -;SKX-NEXT: retq -define <32 x i16> @zext_32x8mem_to_32x16(<32 x i8> *%i , <32 x i1> %mask) nounwind readnone { - %a = load <32 x i8>,<32 x i8> *%i,align 1 - %x = zext <32 x i8> %a to <32 x i16> - %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer - ret <32 x i16> %ret -} - -;SKX-LABEL: sext_32x8mem_to_32x16: -;SKX: ## BB#0: -;SKX-NEXT: vpmovb2m %ymm0, %k1 -;SKX-NEXT: vpmovsxbw (%rdi), %zmm0 {%k1} {z} -;SKX-NEXT: retq -define <32 x i16> @sext_32x8mem_to_32x16(<32 x i8> *%i , <32 x i1> %mask) nounwind readnone { - %a = load <32 x i8>,<32 x i8> *%i,align 1 - %x = sext <32 x i8> %a to <32 x i16> - %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer - ret <32 x i16> %ret -} - -;SKX-LABEL: zext_32x8_to_32x16: -;SKX: ## BB#0: -;SKX-NEXT: vpmovzxbw %ymm0, %zmm0 -;SKX-NEXT: retq -define <32 x i16> @zext_32x8_to_32x16(<32 x i8> %a ) nounwind readnone { - %x = zext <32 x i8> %a to <32 x i16> - ret <32 x i16> %x -} - -;SKX-LABEL: zext_32x8_to_32x16_mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovb2m %ymm1, %k1 -;SKX-NEXT: vpmovzxbw %ymm0, %zmm0 {%k1} {z} -;SKX-NEXT: retq -define <32 x i16> @zext_32x8_to_32x16_mask(<32 x i8> %a ,<32 x i1> %mask) nounwind readnone { - %x = zext <32 x i8> %a to <32 x i16> - %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer - ret <32 x i16> %ret -} - -;SKX-LABEL: sext_32x8_to_32x16: -;SKX: ## BB#0: -;SKX-NEXT: vpmovsxbw %ymm0, %zmm0 -;SKX-NEXT: retq -define <32 x i16> @sext_32x8_to_32x16(<32 x i8> %a ) nounwind readnone { - %x = sext <32 x i8> %a to <32 x i16> - ret <32 x i16> %x -} - -;SKX-LABEL: sext_32x8_to_32x16_mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovb2m %ymm1, %k1 -;SKX-NEXT: vpmovsxbw %ymm0, %zmm0 {%k1} {z} -;SKX-NEXT: retq -define <32 x i16> @sext_32x8_to_32x16_mask(<32 x i8> %a ,<32 x i1> %mask) nounwind readnone { - %x = sext <32 x i8> %a to <32 x i16> - %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer - ret <32 x i16> %ret -} - -;SKX-LABEL: zext_4x8mem_to_4x32: -;SKX: ## BB#0: -;SKX-NEXT: vpmovd2m %xmm0, %k1 -;SKX-NEXT: vpmovzxbd (%rdi), %xmm0 {%k1} {z} -;SKX-NEXT: retq -define <4 x i32> @zext_4x8mem_to_4x32(<4 x i8> *%i , <4 x i1> %mask) nounwind readnone { - %a = load <4 x i8>,<4 x i8> *%i,align 1 - %x = zext <4 x i8> %a to <4 x i32> - %ret = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer - ret <4 x i32> %ret -} - -;SKX-LABEL: sext_4x8mem_to_4x32: -;SKX: ## BB#0: -;SKX-NEXT: vpmovd2m %xmm0, %k1 -;SKX-NEXT: vpmovsxbd (%rdi), %xmm0 {%k1} {z} -;SKX-NEXT: retq -define <4 x i32> @sext_4x8mem_to_4x32(<4 x i8> *%i , <4 x i1> %mask) nounwind readnone { - %a = load <4 x i8>,<4 x i8> *%i,align 1 - %x = sext <4 x i8> %a to <4 x i32> - %ret = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer - ret <4 x i32> %ret -} - -;SKX-LABEL: zext_8x8mem_to_8x32: -;SKX: ## BB#0: -;SKX-NEXT: vpmovw2m %xmm0, %k1 -;SKX-NEXT: vpmovzxbd (%rdi), %ymm0 {%k1} {z} -;SKX-NEXT: retq -define <8 x i32> @zext_8x8mem_to_8x32(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone { - %a = load <8 x i8>,<8 x i8> *%i,align 1 - %x = zext <8 x i8> %a to <8 x i32> - %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer - ret <8 x i32> %ret -} - -;SKX-LABEL: sext_8x8mem_to_8x32: -;SKX: ## BB#0: -;SKX-NEXT: vpmovw2m %xmm0, %k1 -;SKX-NEXT: vpmovsxbd (%rdi), %ymm0 {%k1} {z} -;SKX-NEXT: retq -define <8 x i32> @sext_8x8mem_to_8x32(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone { - %a = load <8 x i8>,<8 x i8> *%i,align 1 - %x = sext <8 x i8> %a to <8 x i32> - %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer - ret <8 x i32> %ret -} - -;KNL-LABEL: zext_16x8mem_to_16x32: -;KNL: vpmovzxbd (%rdi), %zmm0 {%k1} {z} -;KNL-NEXT: retq -define <16 x i32> @zext_16x8mem_to_16x32(<16 x i8> *%i , <16 x i1> %mask) nounwind readnone { - %a = load <16 x i8>,<16 x i8> *%i,align 1 - %x = zext <16 x i8> %a to <16 x i32> - %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer - ret <16 x i32> %ret -} - -;KNL-LABEL: sext_16x8mem_to_16x32: -;KNL: vpmovsxbd (%rdi), %zmm0 {%k1} {z} -;KNL-NEXT: retq -define <16 x i32> @sext_16x8mem_to_16x32(<16 x i8> *%i , <16 x i1> %mask) nounwind readnone { - %a = load <16 x i8>,<16 x i8> *%i,align 1 - %x = sext <16 x i8> %a to <16 x i32> - %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer - ret <16 x i32> %ret -} - -;KNL-LABEL: zext_16x8_to_16x32_mask: -;KNL: vpmovzxbd %xmm0, %zmm0 {%k1} {z} -;KNL-NEXT: retq -define <16 x i32> @zext_16x8_to_16x32_mask(<16 x i8> %a , <16 x i1> %mask) nounwind readnone { - %x = zext <16 x i8> %a to <16 x i32> - %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer - ret <16 x i32> %ret -} - -;KNL-LABEL: sext_16x8_to_16x32_mask: -;KNL: vpmovsxbd %xmm0, %zmm0 {%k1} {z} -;KNL-NEXT: retq -define <16 x i32> @sext_16x8_to_16x32_mask(<16 x i8> %a , <16 x i1> %mask) nounwind readnone { - %x = sext <16 x i8> %a to <16 x i32> - %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer - ret <16 x i32> %ret -} - -; KNL-LABEL: zext_16x8_to_16x32 -; KNL: vpmovzxbd {{.*}}%zmm -; KNL: ret -define <16 x i32> @zext_16x8_to_16x32(<16 x i8> %i) nounwind readnone { - %x = zext <16 x i8> %i to <16 x i32> - ret <16 x i32> %x -} - -; KNL-LABEL: sext_16x8_to_16x32 -; KNL: vpmovsxbd {{.*}}%zmm -; KNL: ret -define <16 x i32> @sext_16x8_to_16x32(<16 x i8> %i) nounwind readnone { - %x = sext <16 x i8> %i to <16 x i32> - ret <16 x i32> %x -} - -;SKX-LABEL: zext_2x8mem_to_2x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovq2m %xmm0, %k1 -;SKX-NEXT: vpmovzxbq (%rdi), %xmm0 {%k1} {z} -;SKX-NEXT: retq -define <2 x i64> @zext_2x8mem_to_2x64(<2 x i8> *%i , <2 x i1> %mask) nounwind readnone { - %a = load <2 x i8>,<2 x i8> *%i,align 1 - %x = zext <2 x i8> %a to <2 x i64> - %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer - ret <2 x i64> %ret -} -;SKX-LABEL: sext_2x8mem_to_2x64mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovq2m %xmm0, %k1 -;SKX-NEXT: vpmovsxbq (%rdi), %xmm0 {%k1} {z} -;SKX-NEXT: retq -define <2 x i64> @sext_2x8mem_to_2x64mask(<2 x i8> *%i , <2 x i1> %mask) nounwind readnone { - %a = load <2 x i8>,<2 x i8> *%i,align 1 - %x = sext <2 x i8> %a to <2 x i64> - %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer - ret <2 x i64> %ret -} -;SKX-LABEL: sext_2x8mem_to_2x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovsxbq (%rdi), %xmm0 -;SKX-NEXT: retq -define <2 x i64> @sext_2x8mem_to_2x64(<2 x i8> *%i) nounwind readnone { - %a = load <2 x i8>,<2 x i8> *%i,align 1 - %x = sext <2 x i8> %a to <2 x i64> - ret <2 x i64> %x -} - -;SKX-LABEL: zext_4x8mem_to_4x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovd2m %xmm0, %k1 -;SKX-NEXT: vpmovzxbq (%rdi), %ymm0 {%k1} {z} -;SKX-NEXT: retq -define <4 x i64> @zext_4x8mem_to_4x64(<4 x i8> *%i , <4 x i1> %mask) nounwind readnone { - %a = load <4 x i8>,<4 x i8> *%i,align 1 - %x = zext <4 x i8> %a to <4 x i64> - %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer - ret <4 x i64> %ret -} - -;SKX-LABEL: sext_4x8mem_to_4x64mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovd2m %xmm0, %k1 -;SKX-NEXT: vpmovsxbq (%rdi), %ymm0 {%k1} {z} -;SKX-NEXT: retq -define <4 x i64> @sext_4x8mem_to_4x64mask(<4 x i8> *%i , <4 x i1> %mask) nounwind readnone { - %a = load <4 x i8>,<4 x i8> *%i,align 1 - %x = sext <4 x i8> %a to <4 x i64> - %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer - ret <4 x i64> %ret -} - -;SKX-LABEL: sext_4x8mem_to_4x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovsxbq (%rdi), %ymm0 -;SKX-NEXT: retq -define <4 x i64> @sext_4x8mem_to_4x64(<4 x i8> *%i) nounwind readnone { - %a = load <4 x i8>,<4 x i8> *%i,align 1 - %x = sext <4 x i8> %a to <4 x i64> - ret <4 x i64> %x -} - -;KNL-LABEL: zext_8x8mem_to_8x64: -;KNL: vpmovzxbq (%rdi), %zmm0 {%k1} {z} -;KNL-NEXT: retq -define <8 x i64> @zext_8x8mem_to_8x64(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone { - %a = load <8 x i8>,<8 x i8> *%i,align 1 - %x = zext <8 x i8> %a to <8 x i64> - %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer - ret <8 x i64> %ret -} - -;KNL-LABEL: sext_8x8mem_to_8x64mask: -;KNL: vpmovsxbq (%rdi), %zmm0 {%k1} {z} -;KNL-NEXT: retq -define <8 x i64> @sext_8x8mem_to_8x64mask(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone { - %a = load <8 x i8>,<8 x i8> *%i,align 1 - %x = sext <8 x i8> %a to <8 x i64> - %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer - ret <8 x i64> %ret -} - -;KNL-LABEL: sext_8x8mem_to_8x64: -;KNL: vpmovsxbq (%rdi), %zmm0 -;KNL-NEXT: retq -define <8 x i64> @sext_8x8mem_to_8x64(<8 x i8> *%i) nounwind readnone { - %a = load <8 x i8>,<8 x i8> *%i,align 1 - %x = sext <8 x i8> %a to <8 x i64> - ret <8 x i64> %x -} - -;SKX-LABEL: zext_4x16mem_to_4x32: -;SKX: ## BB#0: -;SKX-NEXT: vpmovd2m %xmm0, %k1 -;SKX-NEXT: vpmovzxwd (%rdi), %xmm0 {%k1} {z} -;SKX-NEXT: retq -define <4 x i32> @zext_4x16mem_to_4x32(<4 x i16> *%i , <4 x i1> %mask) nounwind readnone { - %a = load <4 x i16>,<4 x i16> *%i,align 1 - %x = zext <4 x i16> %a to <4 x i32> - %ret = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer - ret <4 x i32> %ret -} - -;SKX-LABEL: sext_4x16mem_to_4x32mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovd2m %xmm0, %k1 -;SKX-NEXT: vpmovsxwd (%rdi), %xmm0 {%k1} {z} -;SKX-NEXT: retq -define <4 x i32> @sext_4x16mem_to_4x32mask(<4 x i16> *%i , <4 x i1> %mask) nounwind readnone { - %a = load <4 x i16>,<4 x i16> *%i,align 1 - %x = sext <4 x i16> %a to <4 x i32> - %ret = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer - ret <4 x i32> %ret -} - -;SKX-LABEL: sext_4x16mem_to_4x32: -;SKX: ## BB#0: -;SKX-NEXT: vpmovsxwd (%rdi), %xmm0 -;SKX-NEXT: retq -define <4 x i32> @sext_4x16mem_to_4x32(<4 x i16> *%i) nounwind readnone { - %a = load <4 x i16>,<4 x i16> *%i,align 1 - %x = sext <4 x i16> %a to <4 x i32> - ret <4 x i32> %x -} - - -;SKX-LABEL: zext_8x16mem_to_8x32: -;SKX: ## BB#0: -;SKX-NEXT: vpmovw2m %xmm0, %k1 -;SKX-NEXT: vpmovzxwd (%rdi), %ymm0 {%k1} {z} -;SKX-NEXT: retq -define <8 x i32> @zext_8x16mem_to_8x32(<8 x i16> *%i , <8 x i1> %mask) nounwind readnone { - %a = load <8 x i16>,<8 x i16> *%i,align 1 - %x = zext <8 x i16> %a to <8 x i32> - %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer - ret <8 x i32> %ret -} - -;SKX-LABEL: sext_8x16mem_to_8x32mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovw2m %xmm0, %k1 -;SKX-NEXT: vpmovsxwd (%rdi), %ymm0 {%k1} {z} -;SKX-NEXT: retq -define <8 x i32> @sext_8x16mem_to_8x32mask(<8 x i16> *%i , <8 x i1> %mask) nounwind readnone { - %a = load <8 x i16>,<8 x i16> *%i,align 1 - %x = sext <8 x i16> %a to <8 x i32> - %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer - ret <8 x i32> %ret -} - -;SKX-LABEL: sext_8x16mem_to_8x32: -;SKX: ## BB#0: -;SKX-NEXT: vpmovsxwd (%rdi), %ymm0 -;SKX-NEXT: retq -define <8 x i32> @sext_8x16mem_to_8x32(<8 x i16> *%i) nounwind readnone { - %a = load <8 x i16>,<8 x i16> *%i,align 1 - %x = sext <8 x i16> %a to <8 x i32> - ret <8 x i32> %x -} - -;SKX-LABEL: zext_8x16_to_8x32mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovw2m %xmm1, %k1 -;SKX-NEXT: vpmovzxwd %xmm0, %ymm0 {%k1} {z} -;SKX-NEXT: retq -define <8 x i32> @zext_8x16_to_8x32mask(<8 x i16> %a , <8 x i1> %mask) nounwind readnone { - %x = zext <8 x i16> %a to <8 x i32> - %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer - ret <8 x i32> %ret -} - -;SKX-LABEL: zext_8x16_to_8x32: -;SKX: ## BB#0: -;SKX-NEXT: vpmovzxwd %xmm0, %ymm0 -;SKX-NEXT: retq -define <8 x i32> @zext_8x16_to_8x32(<8 x i16> %a ) nounwind readnone { - %x = zext <8 x i16> %a to <8 x i32> - ret <8 x i32> %x -} - -;SKX-LABEL: zext_16x16mem_to_16x32: -;KNL-LABEL: zext_16x16mem_to_16x32: -;SKX: ## BB#0: -;SKX-NEXT: vpmovb2m %xmm0, %k1 -;SKX-NEXT: vpmovzxwd (%rdi), %zmm0 {%k1} {z} -;KNL: vpmovzxwd (%rdi), %zmm0 {%k1} {z} -;SKX-NEXT: retq -define <16 x i32> @zext_16x16mem_to_16x32(<16 x i16> *%i , <16 x i1> %mask) nounwind readnone { - %a = load <16 x i16>,<16 x i16> *%i,align 1 - %x = zext <16 x i16> %a to <16 x i32> - %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer - ret <16 x i32> %ret -} - -;SKX-LABEL: sext_16x16mem_to_16x32mask: -;KNL-LABEL: sext_16x16mem_to_16x32mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovb2m %xmm0, %k1 -;SKX-NEXT: vpmovsxwd (%rdi), %zmm0 {%k1} {z} -;KNL: vpmovsxwd (%rdi), %zmm0 {%k1} {z} -;SKX-NEXT: retq -define <16 x i32> @sext_16x16mem_to_16x32mask(<16 x i16> *%i , <16 x i1> %mask) nounwind readnone { - %a = load <16 x i16>,<16 x i16> *%i,align 1 - %x = sext <16 x i16> %a to <16 x i32> - %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer - ret <16 x i32> %ret -} - -;SKX-LABEL: sext_16x16mem_to_16x32: -;KNL-LABEL: sext_16x16mem_to_16x32: -;SKX: ## BB#0: -;SKX-NEXT: vpmovsxwd (%rdi), %zmm0 -;KNL: vpmovsxwd (%rdi), %zmm0 -;SKX-NEXT: retq -define <16 x i32> @sext_16x16mem_to_16x32(<16 x i16> *%i) nounwind readnone { - %a = load <16 x i16>,<16 x i16> *%i,align 1 - %x = sext <16 x i16> %a to <16 x i32> - ret <16 x i32> %x -} -;SKX-LABEL: zext_16x16_to_16x32mask: -;KNL-LABEL: zext_16x16_to_16x32mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovb2m %xmm1, %k1 -;SKX-NEXT: vpmovzxwd %ymm0, %zmm0 {%k1} {z} -;KNL: vpmovzxwd %ymm0, %zmm0 {%k1} {z} -;SKX-NEXT: retq -define <16 x i32> @zext_16x16_to_16x32mask(<16 x i16> %a , <16 x i1> %mask) nounwind readnone { - %x = zext <16 x i16> %a to <16 x i32> - %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer - ret <16 x i32> %ret -} - -;SKX-LABEL: zext_16x16_to_16x32: -;KNL-LABEL: zext_16x16_to_16x32: -;SKX: ## BB#0: -;SKX-NEXT: vpmovzxwd %ymm0, %zmm0 -;KNL: vpmovzxwd %ymm0, %zmm0 -;SKX-NEXT: retq -define <16 x i32> @zext_16x16_to_16x32(<16 x i16> %a ) nounwind readnone { - %x = zext <16 x i16> %a to <16 x i32> - ret <16 x i32> %x -} - -;SKX-LABEL: zext_2x16mem_to_2x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovq2m %xmm0, %k1 -;SKX-NEXT: vpmovzxwq (%rdi), %xmm0 {%k1} {z} -;SKX-NEXT: retq -define <2 x i64> @zext_2x16mem_to_2x64(<2 x i16> *%i , <2 x i1> %mask) nounwind readnone { - %a = load <2 x i16>,<2 x i16> *%i,align 1 - %x = zext <2 x i16> %a to <2 x i64> - %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer - ret <2 x i64> %ret -} - -;SKX-LABEL: sext_2x16mem_to_2x64mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovq2m %xmm0, %k1 -;SKX-NEXT: vpmovsxwq (%rdi), %xmm0 {%k1} {z} -;SKX-NEXT: retq -define <2 x i64> @sext_2x16mem_to_2x64mask(<2 x i16> *%i , <2 x i1> %mask) nounwind readnone { - %a = load <2 x i16>,<2 x i16> *%i,align 1 - %x = sext <2 x i16> %a to <2 x i64> - %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer - ret <2 x i64> %ret -} - -;SKX-LABEL: sext_2x16mem_to_2x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovsxwq (%rdi), %xmm0 -;SKX-NEXT: retq -define <2 x i64> @sext_2x16mem_to_2x64(<2 x i16> *%i) nounwind readnone { - %a = load <2 x i16>,<2 x i16> *%i,align 1 - %x = sext <2 x i16> %a to <2 x i64> - ret <2 x i64> %x -} - -;SKX-LABEL: zext_4x16mem_to_4x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovd2m %xmm0, %k1 -;SKX-NEXT: vpmovzxwq (%rdi), %ymm0 {%k1} {z} -;SKX-NEXT: retq -define <4 x i64> @zext_4x16mem_to_4x64(<4 x i16> *%i , <4 x i1> %mask) nounwind readnone { - %a = load <4 x i16>,<4 x i16> *%i,align 1 - %x = zext <4 x i16> %a to <4 x i64> - %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer - ret <4 x i64> %ret -} - -;SKX-LABEL: sext_4x16mem_to_4x64mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovd2m %xmm0, %k1 -;SKX-NEXT: vpmovsxwq (%rdi), %ymm0 {%k1} {z} -;SKX-NEXT: retq -define <4 x i64> @sext_4x16mem_to_4x64mask(<4 x i16> *%i , <4 x i1> %mask) nounwind readnone { - %a = load <4 x i16>,<4 x i16> *%i,align 1 - %x = sext <4 x i16> %a to <4 x i64> - %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer - ret <4 x i64> %ret -} - -;SKX-LABEL: sext_4x16mem_to_4x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovsxwq (%rdi), %ymm0 -;SKX-NEXT: retq -define <4 x i64> @sext_4x16mem_to_4x64(<4 x i16> *%i) nounwind readnone { - %a = load <4 x i16>,<4 x i16> *%i,align 1 - %x = sext <4 x i16> %a to <4 x i64> - ret <4 x i64> %x -} - -;SKX-LABEL: zext_8x16mem_to_8x64: -;KNL-LABEL: zext_8x16mem_to_8x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovw2m %xmm0, %k1 -;SKX-NEXT: vpmovzxwq (%rdi), %zmm0 {%k1} {z} -;KNL: vpmovzxwq (%rdi), %zmm0 {%k1} {z} -;SKX-NEXT: retq -define <8 x i64> @zext_8x16mem_to_8x64(<8 x i16> *%i , <8 x i1> %mask) nounwind readnone { - %a = load <8 x i16>,<8 x i16> *%i,align 1 - %x = zext <8 x i16> %a to <8 x i64> - %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer - ret <8 x i64> %ret -} - -;SKX-LABEL: sext_8x16mem_to_8x64mask: -;KNL-LABEL: sext_8x16mem_to_8x64mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovw2m %xmm0, %k1 -;SKX-NEXT: vpmovsxwq (%rdi), %zmm0 {%k1} {z} -;KNL: vpmovsxwq (%rdi), %zmm0 {%k1} {z} -;SKX-NEXT: retq -define <8 x i64> @sext_8x16mem_to_8x64mask(<8 x i16> *%i , <8 x i1> %mask) nounwind readnone { - %a = load <8 x i16>,<8 x i16> *%i,align 1 - %x = sext <8 x i16> %a to <8 x i64> - %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer - ret <8 x i64> %ret -} - -;SKX-LABEL: sext_8x16mem_to_8x64: -;KNL-LABEL: sext_8x16mem_to_8x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovsxwq (%rdi), %zmm0 -;KNL: vpmovsxwq (%rdi), %zmm0 -;SKX-NEXT: retq -define <8 x i64> @sext_8x16mem_to_8x64(<8 x i16> *%i) nounwind readnone { - %a = load <8 x i16>,<8 x i16> *%i,align 1 - %x = sext <8 x i16> %a to <8 x i64> - ret <8 x i64> %x -} - -;SKX-LABEL: zext_8x16_to_8x64mask: -;KNL-LABEL: zext_8x16_to_8x64mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovw2m %xmm1, %k1 -;SKX-NEXT: vpmovzxwq %xmm0, %zmm0 {%k1} {z} -;KNL: vpmovzxwq %xmm0, %zmm0 {%k1} {z} -;SKX-NEXT: retq -define <8 x i64> @zext_8x16_to_8x64mask(<8 x i16> %a , <8 x i1> %mask) nounwind readnone { - %x = zext <8 x i16> %a to <8 x i64> - %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer - ret <8 x i64> %ret -} - -;SKX-LABEL: zext_8x16_to_8x64: -;KNL-LABEL: zext_8x16_to_8x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovzxwq %xmm0, %zmm0 -;KNL: vpmovzxwq %xmm0, %zmm0 -;SKX-NEXT: retq -; KNL: ret -define <8 x i64> @zext_8x16_to_8x64(<8 x i16> %a) nounwind readnone { - %ret = zext <8 x i16> %a to <8 x i64> - ret <8 x i64> %ret -} - -;SKX-LABEL: zext_2x32mem_to_2x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovq2m %xmm0, %k1 -;SKX-NEXT: vpmovzxdq (%rdi), %xmm0 {%k1} {z} -;SKX-NEXT: retq -define <2 x i64> @zext_2x32mem_to_2x64(<2 x i32> *%i , <2 x i1> %mask) nounwind readnone { - %a = load <2 x i32>,<2 x i32> *%i,align 1 - %x = zext <2 x i32> %a to <2 x i64> - %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer - ret <2 x i64> %ret -} - -;SKX-LABEL: sext_2x32mem_to_2x64mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovq2m %xmm0, %k1 -;SKX-NEXT: vpmovsxdq (%rdi), %xmm0 {%k1} {z} -;SKX-NEXT: retq -define <2 x i64> @sext_2x32mem_to_2x64mask(<2 x i32> *%i , <2 x i1> %mask) nounwind readnone { - %a = load <2 x i32>,<2 x i32> *%i,align 1 - %x = sext <2 x i32> %a to <2 x i64> - %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer - ret <2 x i64> %ret -} - -;SKX-LABEL: sext_2x32mem_to_2x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovsxdq (%rdi), %xmm0 -;SKX-NEXT: retq -define <2 x i64> @sext_2x32mem_to_2x64(<2 x i32> *%i) nounwind readnone { - %a = load <2 x i32>,<2 x i32> *%i,align 1 - %x = sext <2 x i32> %a to <2 x i64> - ret <2 x i64> %x -} - -;SKX-LABEL: zext_4x32mem_to_4x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovd2m %xmm0, %k1 -;SKX-NEXT: vpmovzxdq (%rdi), %ymm0 {%k1} {z} -;SKX-NEXT: retq -define <4 x i64> @zext_4x32mem_to_4x64(<4 x i32> *%i , <4 x i1> %mask) nounwind readnone { - %a = load <4 x i32>,<4 x i32> *%i,align 1 - %x = zext <4 x i32> %a to <4 x i64> - %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer - ret <4 x i64> %ret -} - -;SKX-LABEL: sext_4x32mem_to_4x64mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovd2m %xmm0, %k1 -;SKX-NEXT: vpmovsxdq (%rdi), %ymm0 {%k1} {z} -;SKX-NEXT: retq -define <4 x i64> @sext_4x32mem_to_4x64mask(<4 x i32> *%i , <4 x i1> %mask) nounwind readnone { - %a = load <4 x i32>,<4 x i32> *%i,align 1 - %x = sext <4 x i32> %a to <4 x i64> - %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer - ret <4 x i64> %ret -} - -;SKX-LABEL: sext_4x32mem_to_4x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovsxdq (%rdi), %ymm0 -;SKX-NEXT: retq -define <4 x i64> @sext_4x32mem_to_4x64(<4 x i32> *%i) nounwind readnone { - %a = load <4 x i32>,<4 x i32> *%i,align 1 - %x = sext <4 x i32> %a to <4 x i64> - ret <4 x i64> %x -} - -;SKX-LABEL: sext_4x32_to_4x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovsxdq %xmm0, %ymm0 -;SKX-NEXT: retq -define <4 x i64> @sext_4x32_to_4x64(<4 x i32> %a) nounwind readnone { - %x = sext <4 x i32> %a to <4 x i64> - ret <4 x i64> %x -} - -;SKX-LABEL: zext_4x32_to_4x64mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovd2m %xmm1, %k1 -;SKX-NEXT: vpmovzxdq %xmm0, %ymm0 {%k1} {z} -;SKX-NEXT: retq -define <4 x i64> @zext_4x32_to_4x64mask(<4 x i32> %a , <4 x i1> %mask) nounwind readnone { - %x = zext <4 x i32> %a to <4 x i64> - %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer - ret <4 x i64> %ret -} - -;SKX-LABEL: zext_8x32mem_to_8x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovw2m %xmm0, %k1 -;SKX-NEXT: vpmovzxdq (%rdi), %zmm0 {%k1} {z} -;SKX-NEXT: retq -define <8 x i64> @zext_8x32mem_to_8x64(<8 x i32> *%i , <8 x i1> %mask) nounwind readnone { - %a = load <8 x i32>,<8 x i32> *%i,align 1 - %x = zext <8 x i32> %a to <8 x i64> - %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer - ret <8 x i64> %ret -} - -;SKX-LABEL: sext_8x32mem_to_8x64mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovw2m %xmm0, %k1 -;SKX-NEXT: vpmovsxdq (%rdi), %zmm0 {%k1} {z} -;SKX-NEXT: retq -define <8 x i64> @sext_8x32mem_to_8x64mask(<8 x i32> *%i , <8 x i1> %mask) nounwind readnone { - %a = load <8 x i32>,<8 x i32> *%i,align 1 - %x = sext <8 x i32> %a to <8 x i64> - %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer - ret <8 x i64> %ret -} - -;SKX-LABEL: sext_8x32mem_to_8x64: -;KNL-LABEL: sext_8x32mem_to_8x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovsxdq (%rdi), %zmm0 -;KNL: vpmovsxdq (%rdi), %zmm0 -;SKX-NEXT: retq -define <8 x i64> @sext_8x32mem_to_8x64(<8 x i32> *%i) nounwind readnone { - %a = load <8 x i32>,<8 x i32> *%i,align 1 - %x = sext <8 x i32> %a to <8 x i64> - ret <8 x i64> %x -} - -;SKX-LABEL: sext_8x32_to_8x64: -;KNL-LABEL: sext_8x32_to_8x64: -;SKX: ## BB#0: -;SKX-NEXT: vpmovsxdq %ymm0, %zmm0 -;KNL: vpmovsxdq %ymm0, %zmm0 -;SKX-NEXT: retq -define <8 x i64> @sext_8x32_to_8x64(<8 x i32> %a) nounwind readnone { - %x = sext <8 x i32> %a to <8 x i64> - ret <8 x i64> %x -} - -;SKX-LABEL: zext_8x32_to_8x64mask: -;KNL-LABEL: zext_8x32_to_8x64mask: -;SKX: ## BB#0: -;SKX-NEXT: vpmovw2m %xmm1, %k1 -;SKX-NEXT: vpmovzxdq %ymm0, %zmm0 {%k1} {z} -;KNL: vpmovzxdq %ymm0, %zmm0 {%k1} {z} -;SKX-NEXT: retq -define <8 x i64> @zext_8x32_to_8x64mask(<8 x i32> %a , <8 x i1> %mask) nounwind readnone { - %x = zext <8 x i32> %a to <8 x i64> - %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer - ret <8 x i64> %ret -} -;KNL-LABEL: fptrunc_test -;KNL: vcvtpd2ps {{.*}}%zmm -;KNL: ret -define <8 x float> @fptrunc_test(<8 x double> %a) nounwind readnone { - %b = fptrunc <8 x double> %a to <8 x float> - ret <8 x float> %b -} - -;KNL-LABEL: fpext_test -;KNL: vcvtps2pd {{.*}}%zmm -;KNL: ret -define <8 x double> @fpext_test(<8 x float> %a) nounwind readnone { - %b = fpext <8 x float> %a to <8 x double> - ret <8 x double> %b -} - -; KNL-LABEL: zext_16i1_to_16xi32 -; KNL: vpbroadcastd LCP{{.*}}(%rip), %zmm0 {%k1} {z} -; KNL: ret -define <16 x i32> @zext_16i1_to_16xi32(i16 %b) { - %a = bitcast i16 %b to <16 x i1> - %c = zext <16 x i1> %a to <16 x i32> - ret <16 x i32> %c -} - -; KNL-LABEL: zext_8i1_to_8xi64 -; KNL: vpbroadcastq LCP{{.*}}(%rip), %zmm0 {%k1} {z} -; KNL: ret -define <8 x i64> @zext_8i1_to_8xi64(i8 %b) { - %a = bitcast i8 %b to <8 x i1> - %c = zext <8 x i1> %a to <8 x i64> - ret <8 x i64> %c -} - -; KNL-LABEL: trunc_16i8_to_16i1 -; KNL: vpmovsxbd -; KNL: vpandd -; KNL: vptestmd -; KNL: ret -; SKX-LABEL: trunc_16i8_to_16i1 -; SKX: vpmovb2m %xmm -define i16 @trunc_16i8_to_16i1(<16 x i8> %a) { - %mask_b = trunc <16 x i8>%a to <16 x i1> - %mask = bitcast <16 x i1> %mask_b to i16 - ret i16 %mask -} - -; KNL-LABEL: trunc_16i32_to_16i1 -; KNL: vpandd -; KNL: vptestmd -; KNL: ret -; SKX-LABEL: trunc_16i32_to_16i1 -; SKX: vpmovd2m %zmm -define i16 @trunc_16i32_to_16i1(<16 x i32> %a) { - %mask_b = trunc <16 x i32>%a to <16 x i1> - %mask = bitcast <16 x i1> %mask_b to i16 - ret i16 %mask -} - -; SKX-LABEL: trunc_4i32_to_4i1 -; SKX: vpmovd2m %xmm -; SKX: kandw -; SKX: vpmovm2d -define <4 x i32> @trunc_4i32_to_4i1(<4 x i32> %a, <4 x i32> %b) { - %mask_a = trunc <4 x i32>%a to <4 x i1> - %mask_b = trunc <4 x i32>%b to <4 x i1> - %a_and_b = and <4 x i1>%mask_a, %mask_b - %res = sext <4 x i1>%a_and_b to <4 x i32> - ret <4 x i32>%res -} - -; KNL-LABEL: trunc_8i16_to_8i1 -; KNL: vpmovsxwq -; KNL: vpandq LCP{{.*}}(%rip){1to8} -; KNL: vptestmq -; KNL: ret - -; SKX-LABEL: trunc_8i16_to_8i1 -; SKX: vpmovw2m %xmm -define i8 @trunc_8i16_to_8i1(<8 x i16> %a) { - %mask_b = trunc <8 x i16>%a to <8 x i1> - %mask = bitcast <8 x i1> %mask_b to i8 - ret i8 %mask -} - -; KNL-LABEL: sext_8i1_8i32 -; KNL: vpbroadcastq LCP{{.*}}(%rip), %zmm0 {%k1} {z} -; SKX: vpmovm2d -; KNL: ret -define <8 x i32> @sext_8i1_8i32(<8 x i32> %a1, <8 x i32> %a2) nounwind { - %x = icmp slt <8 x i32> %a1, %a2 - %x1 = xor <8 x i1>%x, - %y = sext <8 x i1> %x1 to <8 x i32> - ret <8 x i32> %y -} - -; KNL-LABEL: trunc_v16i32_to_v16i16 -; KNL: vpmovdw -; KNL: ret -define <16 x i16> @trunc_v16i32_to_v16i16(<16 x i32> %x) { - %1 = trunc <16 x i32> %x to <16 x i16> - ret <16 x i16> %1 -} - -; KNL-LABEL: trunc_i32_to_i1 -; KNL: movw $-4, %ax -; KNL: kmovw %eax, %k1 -; KNL: korw -define i16 @trunc_i32_to_i1(i32 %a) { - %a_i = trunc i32 %a to i1 - %maskv = insertelement <16 x i1> , i1 %a_i, i32 0 - %res = bitcast <16 x i1> %maskv to i16 - ret i16 %res -} - -; KNL-LABEL: sext_8i1_8i16 -; SKX: vpmovm2w -; KNL: ret -define <8 x i16> @sext_8i1_8i16(<8 x i32> %a1, <8 x i32> %a2) nounwind { - %x = icmp slt <8 x i32> %a1, %a2 - %y = sext <8 x i1> %x to <8 x i16> - ret <8 x i16> %y -} - -; KNL-LABEL: sext_16i1_16i32 -; SKX: vpmovm2d -; KNL: ret -define <16 x i32> @sext_16i1_16i32(<16 x i32> %a1, <16 x i32> %a2) nounwind { - %x = icmp slt <16 x i32> %a1, %a2 - %y = sext <16 x i1> %x to <16 x i32> - ret <16 x i32> %y -} - -; KNL-LABEL: sext_8i1_8i64 -; SKX: vpmovm2q -; KNL: ret -define <8 x i64> @sext_8i1_8i64(<8 x i32> %a1, <8 x i32> %a2) nounwind { - %x = icmp slt <8 x i32> %a1, %a2 - %y = sext <8 x i1> %x to <8 x i64> - ret <8 x i64> %y -} - -; KNL-LABEL: @extload_v8i64 -; KNL: vpmovsxbq -define void @extload_v8i64(<8 x i8>* %a, <8 x i64>* %res) { - %sign_load = load <8 x i8>, <8 x i8>* %a - %c = sext <8 x i8> %sign_load to <8 x i64> - store <8 x i64> %c, <8 x i64>* %res - ret void -} - -;SKX-LABEL: test21: -;SKX: vmovdqu16 %zmm0, %zmm3 {%k1} -;SKX-NEXT: kshiftrq $32, %k1, %k1 -;SKX-NEXT: vmovdqu16 %zmm1, %zmm2 {%k1} -define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone { - %ret = select <64 x i1> %mask, <64 x i16> %x, <64 x i16> zeroinitializer - ret <64 x i16> %ret -} - diff --git a/test/CodeGen/X86/avx512-trunc.ll b/test/CodeGen/X86/avx512-trunc.ll new file mode 100644 index 000000000000..e4e5c2b8a1d5 --- /dev/null +++ b/test/CodeGen/X86/avx512-trunc.ll @@ -0,0 +1,488 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=KNL +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f -mattr=+avx512vl -mattr=+avx512bw -mattr=+avx512dq | FileCheck %s --check-prefix=ALL --check-prefix=SKX + + attributes #0 = { nounwind } + +define <16 x i8> @trunc_16x32_to_16x8(<16 x i32> %i) #0 { +; ALL-LABEL: trunc_16x32_to_16x8: +; ALL: ## BB#0: +; ALL-NEXT: vpmovdb %zmm0, %xmm0 +; ALL-NEXT: retq + %x = trunc <16 x i32> %i to <16 x i8> + ret <16 x i8> %x +} + +define <8 x i16> @trunc_8x64_to_8x16(<8 x i64> %i) #0 { +; ALL-LABEL: trunc_8x64_to_8x16: +; ALL: ## BB#0: +; ALL-NEXT: vpmovqw %zmm0, %xmm0 +; ALL-NEXT: retq + %x = trunc <8 x i64> %i to <8 x i16> + ret <8 x i16> %x +} + +define <16 x i16> @trunc_v16i32_to_v16i16(<16 x i32> %x) #0 { +; ALL-LABEL: trunc_v16i32_to_v16i16: +; ALL: ## BB#0: +; ALL-NEXT: vpmovdw %zmm0, %ymm0 +; ALL-NEXT: retq + %1 = trunc <16 x i32> %x to <16 x i16> + ret <16 x i16> %1 +} + +define <8 x i8> @trunc_qb_512(<8 x i64> %i) #0 { +; ALL-LABEL: trunc_qb_512: +; ALL: ## BB#0: +; ALL-NEXT: vpmovqw %zmm0, %xmm0 +; ALL-NEXT: retq + %x = trunc <8 x i64> %i to <8 x i8> + ret <8 x i8> %x +} + +define void @trunc_qb_512_mem(<8 x i64> %i, <8 x i8>* %res) #0 { +; ALL-LABEL: trunc_qb_512_mem: +; ALL: ## BB#0: +; ALL-NEXT: vpmovqb %zmm0, (%rdi) +; ALL-NEXT: retq + %x = trunc <8 x i64> %i to <8 x i8> + store <8 x i8> %x, <8 x i8>* %res + ret void +} + +define <4 x i8> @trunc_qb_256(<4 x i64> %i) #0 { +; KNL-LABEL: trunc_qb_256: +; KNL: ## BB#0: +; KNL-NEXT: vpmovqd %zmm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_qb_256: +; SKX: ## BB#0: +; SKX-NEXT: vpmovqd %ymm0, %xmm0 +; SKX-NEXT: retq + %x = trunc <4 x i64> %i to <4 x i8> + ret <4 x i8> %x +} + +define void @trunc_qb_256_mem(<4 x i64> %i, <4 x i8>* %res) #0 { +; KNL-LABEL: trunc_qb_256_mem: +; KNL: ## BB#0: +; KNL-NEXT: vpmovqd %zmm0, %ymm0 +; KNL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u] +; KNL-NEXT: vmovd %xmm0, (%rdi) +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_qb_256_mem: +; SKX: ## BB#0: +; SKX-NEXT: vpmovqb %ymm0, (%rdi) +; SKX-NEXT: retq + %x = trunc <4 x i64> %i to <4 x i8> + store <4 x i8> %x, <4 x i8>* %res + ret void +} + +define <2 x i8> @trunc_qb_128(<2 x i64> %i) #0 { +; ALL-LABEL: trunc_qb_128: +; ALL: ## BB#0: +; ALL-NEXT: retq + %x = trunc <2 x i64> %i to <2 x i8> + ret <2 x i8> %x +} + +define void @trunc_qb_128_mem(<2 x i64> %i, <2 x i8>* %res) #0 { +; KNL-LABEL: trunc_qb_128_mem: +; KNL: ## BB#0: +; KNL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u] +; KNL-NEXT: vmovd %xmm0, %eax +; KNL-NEXT: movw %ax, (%rdi) +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_qb_128_mem: +; SKX: ## BB#0: +; SKX-NEXT: vpmovqb %xmm0, (%rdi) +; SKX-NEXT: retq + %x = trunc <2 x i64> %i to <2 x i8> + store <2 x i8> %x, <2 x i8>* %res + ret void +} + +define <8 x i16> @trunc_qw_512(<8 x i64> %i) #0 { +; ALL-LABEL: trunc_qw_512: +; ALL: ## BB#0: +; ALL-NEXT: vpmovqw %zmm0, %xmm0 +; ALL-NEXT: retq + %x = trunc <8 x i64> %i to <8 x i16> + ret <8 x i16> %x +} + +define void @trunc_qw_512_mem(<8 x i64> %i, <8 x i16>* %res) #0 { +; ALL-LABEL: trunc_qw_512_mem: +; ALL: ## BB#0: +; ALL-NEXT: vpmovqw %zmm0, (%rdi) +; ALL-NEXT: retq + %x = trunc <8 x i64> %i to <8 x i16> + store <8 x i16> %x, <8 x i16>* %res + ret void +} + +define <4 x i16> @trunc_qw_256(<4 x i64> %i) #0 { +; KNL-LABEL: trunc_qw_256: +; KNL: ## BB#0: +; KNL-NEXT: vpmovqd %zmm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_qw_256: +; SKX: ## BB#0: +; SKX-NEXT: vpmovqd %ymm0, %xmm0 +; SKX-NEXT: retq + %x = trunc <4 x i64> %i to <4 x i16> + ret <4 x i16> %x +} + +define void @trunc_qw_256_mem(<4 x i64> %i, <4 x i16>* %res) #0 { +; KNL-LABEL: trunc_qw_256_mem: +; KNL: ## BB#0: +; KNL-NEXT: vpmovqd %zmm0, %ymm0 +; KNL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] +; KNL-NEXT: vmovq %xmm0, (%rdi) +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_qw_256_mem: +; SKX: ## BB#0: +; SKX-NEXT: vpmovqw %ymm0, (%rdi) +; SKX-NEXT: retq + %x = trunc <4 x i64> %i to <4 x i16> + store <4 x i16> %x, <4 x i16>* %res + ret void +} + +define <2 x i16> @trunc_qw_128(<2 x i64> %i) #0 { +; ALL-LABEL: trunc_qw_128: +; ALL: ## BB#0: +; ALL-NEXT: retq + %x = trunc <2 x i64> %i to <2 x i16> + ret <2 x i16> %x +} + +define void @trunc_qw_128_mem(<2 x i64> %i, <2 x i16>* %res) #0 { +; KNL-LABEL: trunc_qw_128_mem: +; KNL: ## BB#0: +; KNL-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; KNL-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] +; KNL-NEXT: vmovd %xmm0, (%rdi) +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_qw_128_mem: +; SKX: ## BB#0: +; SKX-NEXT: vpmovqw %xmm0, (%rdi) +; SKX-NEXT: retq + %x = trunc <2 x i64> %i to <2 x i16> + store <2 x i16> %x, <2 x i16>* %res + ret void +} + +define <8 x i32> @trunc_qd_512(<8 x i64> %i) #0 { +; ALL-LABEL: trunc_qd_512: +; ALL: ## BB#0: +; ALL-NEXT: vpmovqd %zmm0, %ymm0 +; ALL-NEXT: retq + %x = trunc <8 x i64> %i to <8 x i32> + ret <8 x i32> %x +} + +define void @trunc_qd_512_mem(<8 x i64> %i, <8 x i32>* %res) #0 { +; ALL-LABEL: trunc_qd_512_mem: +; ALL: ## BB#0: +; ALL-NEXT: vpmovqd %zmm0, (%rdi) +; ALL-NEXT: retq + %x = trunc <8 x i64> %i to <8 x i32> + store <8 x i32> %x, <8 x i32>* %res + ret void +} + +define <4 x i32> @trunc_qd_256(<4 x i64> %i) #0 { +; KNL-LABEL: trunc_qd_256: +; KNL: ## BB#0: +; KNL-NEXT: vpmovqd %zmm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_qd_256: +; SKX: ## BB#0: +; SKX-NEXT: vpmovqd %ymm0, %xmm0 +; SKX-NEXT: retq + %x = trunc <4 x i64> %i to <4 x i32> + ret <4 x i32> %x +} + +define void @trunc_qd_256_mem(<4 x i64> %i, <4 x i32>* %res) #0 { +; KNL-LABEL: trunc_qd_256_mem: +; KNL: ## BB#0: +; KNL-NEXT: vpmovqd %zmm0, %ymm0 +; KNL-NEXT: vmovaps %xmm0, (%rdi) +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_qd_256_mem: +; SKX: ## BB#0: +; SKX-NEXT: vpmovqd %ymm0, (%rdi) +; SKX-NEXT: retq + %x = trunc <4 x i64> %i to <4 x i32> + store <4 x i32> %x, <4 x i32>* %res + ret void +} + +define <2 x i32> @trunc_qd_128(<2 x i64> %i) #0 { +; ALL-LABEL: trunc_qd_128: +; ALL: ## BB#0: +; ALL-NEXT: retq + %x = trunc <2 x i64> %i to <2 x i32> + ret <2 x i32> %x +} + +define void @trunc_qd_128_mem(<2 x i64> %i, <2 x i32>* %res) #0 { +; KNL-LABEL: trunc_qd_128_mem: +; KNL: ## BB#0: +; KNL-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; KNL-NEXT: vmovq %xmm0, (%rdi) +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_qd_128_mem: +; SKX: ## BB#0: +; SKX-NEXT: vpmovqd %xmm0, (%rdi) +; SKX-NEXT: retq + %x = trunc <2 x i64> %i to <2 x i32> + store <2 x i32> %x, <2 x i32>* %res + ret void +} + +define <16 x i8> @trunc_db_512(<16 x i32> %i) #0 { +; ALL-LABEL: trunc_db_512: +; ALL: ## BB#0: +; ALL-NEXT: vpmovdb %zmm0, %xmm0 +; ALL-NEXT: retq + %x = trunc <16 x i32> %i to <16 x i8> + ret <16 x i8> %x +} + +define void @trunc_db_512_mem(<16 x i32> %i, <16 x i8>* %res) #0 { +; ALL-LABEL: trunc_db_512_mem: +; ALL: ## BB#0: +; ALL-NEXT: vpmovdb %zmm0, (%rdi) +; ALL-NEXT: retq + %x = trunc <16 x i32> %i to <16 x i8> + store <16 x i8> %x, <16 x i8>* %res + ret void +} + +define <8 x i8> @trunc_db_256(<8 x i32> %i) #0 { +; KNL-LABEL: trunc_db_256: +; KNL: ## BB#0: +; KNL-NEXT: vpmovdw %zmm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_db_256: +; SKX: ## BB#0: +; SKX-NEXT: vpmovdw %ymm0, %xmm0 +; SKX-NEXT: retq + %x = trunc <8 x i32> %i to <8 x i8> + ret <8 x i8> %x +} + +define void @trunc_db_256_mem(<8 x i32> %i, <8 x i8>* %res) #0 { +; KNL-LABEL: trunc_db_256_mem: +; KNL: ## BB#0: +; KNL-NEXT: vpmovdw %zmm0, %ymm0 +; KNL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u] +; KNL-NEXT: vmovq %xmm0, (%rdi) +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_db_256_mem: +; SKX: ## BB#0: +; SKX-NEXT: vpmovdb %ymm0, (%rdi) +; SKX-NEXT: retq + %x = trunc <8 x i32> %i to <8 x i8> + store <8 x i8> %x, <8 x i8>* %res + ret void +} + +define <4 x i8> @trunc_db_128(<4 x i32> %i) #0 { +; ALL-LABEL: trunc_db_128: +; ALL: ## BB#0: +; ALL-NEXT: retq + %x = trunc <4 x i32> %i to <4 x i8> + ret <4 x i8> %x +} + +define void @trunc_db_128_mem(<4 x i32> %i, <4 x i8>* %res) #0 { +; KNL-LABEL: trunc_db_128_mem: +; KNL: ## BB#0: +; KNL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u] +; KNL-NEXT: vmovd %xmm0, (%rdi) +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_db_128_mem: +; SKX: ## BB#0: +; SKX-NEXT: vpmovdb %xmm0, (%rdi) +; SKX-NEXT: retq + %x = trunc <4 x i32> %i to <4 x i8> + store <4 x i8> %x, <4 x i8>* %res + ret void +} + +define <16 x i16> @trunc_dw_512(<16 x i32> %i) #0 { +; ALL-LABEL: trunc_dw_512: +; ALL: ## BB#0: +; ALL-NEXT: vpmovdw %zmm0, %ymm0 +; ALL-NEXT: retq + %x = trunc <16 x i32> %i to <16 x i16> + ret <16 x i16> %x +} + +define void @trunc_dw_512_mem(<16 x i32> %i, <16 x i16>* %res) #0 { +; ALL-LABEL: trunc_dw_512_mem: +; ALL: ## BB#0: +; ALL-NEXT: vpmovdw %zmm0, (%rdi) +; ALL-NEXT: retq + %x = trunc <16 x i32> %i to <16 x i16> + store <16 x i16> %x, <16 x i16>* %res + ret void +} + +define <8 x i16> @trunc_dw_256(<8 x i32> %i) #0 { +; KNL-LABEL: trunc_dw_256: +; KNL: ## BB#0: +; KNL-NEXT: vpmovdw %zmm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_dw_256: +; SKX: ## BB#0: +; SKX-NEXT: vpmovdw %ymm0, %xmm0 +; SKX-NEXT: retq + %x = trunc <8 x i32> %i to <8 x i16> + ret <8 x i16> %x +} + +define void @trunc_dw_256_mem(<8 x i32> %i, <8 x i16>* %res) #0 { +; KNL-LABEL: trunc_dw_256_mem: +; KNL: ## BB#0: +; KNL-NEXT: vpmovdw %zmm0, %ymm0 +; KNL-NEXT: vmovaps %xmm0, (%rdi) +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_dw_256_mem: +; SKX: ## BB#0: +; SKX-NEXT: vpmovdw %ymm0, (%rdi) +; SKX-NEXT: retq + %x = trunc <8 x i32> %i to <8 x i16> + store <8 x i16> %x, <8 x i16>* %res + ret void +} + +define void @trunc_dw_128_mem(<4 x i32> %i, <4 x i16>* %res) #0 { +; KNL-LABEL: trunc_dw_128_mem: +; KNL: ## BB#0: +; KNL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] +; KNL-NEXT: vmovq %xmm0, (%rdi) +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_dw_128_mem: +; SKX: ## BB#0: +; SKX-NEXT: vpmovdw %xmm0, (%rdi) +; SKX-NEXT: retq + %x = trunc <4 x i32> %i to <4 x i16> + store <4 x i16> %x, <4 x i16>* %res + ret void +} + +define <32 x i8> @trunc_wb_512(<32 x i16> %i) #0 { +; KNL-LABEL: trunc_wb_512: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwd %ymm0, %zmm0 +; KNL-NEXT: vpmovdb %zmm0, %xmm0 +; KNL-NEXT: vpmovsxwd %ymm1, %zmm1 +; KNL-NEXT: vpmovdb %zmm1, %xmm1 +; KNL-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_wb_512: +; SKX: ## BB#0: +; SKX-NEXT: vpmovwb %zmm0, %ymm0 +; SKX-NEXT: retq + %x = trunc <32 x i16> %i to <32 x i8> + ret <32 x i8> %x +} + +define void @trunc_wb_512_mem(<32 x i16> %i, <32 x i8>* %res) #0 { +; KNL-LABEL: trunc_wb_512_mem: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwd %ymm0, %zmm0 +; KNL-NEXT: vpmovdb %zmm0, %xmm0 +; KNL-NEXT: vpmovsxwd %ymm1, %zmm1 +; KNL-NEXT: vpmovdb %zmm1, %xmm1 +; KNL-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; KNL-NEXT: vmovaps %ymm0, (%rdi) +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_wb_512_mem: +; SKX: ## BB#0: +; SKX-NEXT: vpmovwb %zmm0, (%rdi) +; SKX-NEXT: retq + %x = trunc <32 x i16> %i to <32 x i8> + store <32 x i8> %x, <32 x i8>* %res + ret void +} + +define <16 x i8> @trunc_wb_256(<16 x i16> %i) #0 { +; KNL-LABEL: trunc_wb_256: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwd %ymm0, %zmm0 +; KNL-NEXT: vpmovdb %zmm0, %xmm0 +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_wb_256: +; SKX: ## BB#0: +; SKX-NEXT: vpmovwb %ymm0, %xmm0 +; SKX-NEXT: retq + %x = trunc <16 x i16> %i to <16 x i8> + ret <16 x i8> %x +} + +define void @trunc_wb_256_mem(<16 x i16> %i, <16 x i8>* %res) #0 { +; KNL-LABEL: trunc_wb_256_mem: +; KNL: ## BB#0: +; KNL-NEXT: vpmovsxwd %ymm0, %zmm0 +; KNL-NEXT: vpmovdb %zmm0, %xmm0 +; KNL-NEXT: vmovaps %xmm0, (%rdi) +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_wb_256_mem: +; SKX: ## BB#0: +; SKX-NEXT: vpmovwb %ymm0, (%rdi) +; SKX-NEXT: retq + %x = trunc <16 x i16> %i to <16 x i8> + store <16 x i8> %x, <16 x i8>* %res + ret void +} + +define <8 x i8> @trunc_wb_128(<8 x i16> %i) #0 { +; ALL-LABEL: trunc_wb_128: +; ALL: ## BB#0: +; ALL-NEXT: retq + %x = trunc <8 x i16> %i to <8 x i8> + ret <8 x i8> %x +} + +define void @trunc_wb_128_mem(<8 x i16> %i, <8 x i8>* %res) #0 { +; KNL-LABEL: trunc_wb_128_mem: +; KNL: ## BB#0: +; KNL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u] +; KNL-NEXT: vmovq %xmm0, (%rdi) +; KNL-NEXT: retq +; +; SKX-LABEL: trunc_wb_128_mem: +; SKX: ## BB#0: +; SKX-NEXT: vpmovwb %xmm0, (%rdi) +; SKX-NEXT: retq + %x = trunc <8 x i16> %i to <8 x i8> + store <8 x i8> %x, <8 x i8>* %res + ret void +} diff --git a/test/CodeGen/X86/avx512-vbroadcast.ll b/test/CodeGen/X86/avx512-vbroadcast.ll index 854f1019f0f8..4f679f9aca6f 100644 --- a/test/CodeGen/X86/avx512-vbroadcast.ll +++ b/test/CodeGen/X86/avx512-vbroadcast.ll @@ -1,47 +1,54 @@ -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512F +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW define <16 x i32> @_inreg16xi32(i32 %a) { -; CHECK-LABEL: _inreg16xi32: -; CHECK: ## BB#0: -; CHECK-NEXT: vpbroadcastd %edi, %zmm0 -; CHECK-NEXT: retq +; ALL-LABEL: _inreg16xi32: +; ALL: # BB#0: +; ALL-NEXT: vpbroadcastd %edi, %zmm0 +; ALL-NEXT: retq %b = insertelement <16 x i32> undef, i32 %a, i32 0 %c = shufflevector <16 x i32> %b, <16 x i32> undef, <16 x i32> zeroinitializer ret <16 x i32> %c } define <8 x i64> @_inreg8xi64(i64 %a) { -; CHECK-LABEL: _inreg8xi64: -; CHECK: ## BB#0: -; CHECK-NEXT: vpbroadcastq %rdi, %zmm0 -; CHECK-NEXT: retq +; ALL-LABEL: _inreg8xi64: +; ALL: # BB#0: +; ALL-NEXT: vpbroadcastq %rdi, %zmm0 +; ALL-NEXT: retq %b = insertelement <8 x i64> undef, i64 %a, i32 0 %c = shufflevector <8 x i64> %b, <8 x i64> undef, <8 x i32> zeroinitializer ret <8 x i64> %c } -;CHECK-LABEL: _ss16xfloat_v4 -;CHECK: vbroadcastss %xmm0, %zmm0 -;CHECK: ret define <16 x float> @_ss16xfloat_v4(<4 x float> %a) { +; ALL-LABEL: _ss16xfloat_v4: +; ALL: # BB#0: +; ALL-NEXT: vbroadcastss %xmm0, %zmm0 +; ALL-NEXT: retq %b = shufflevector <4 x float> %a, <4 x float> undef, <16 x i32> zeroinitializer ret <16 x float> %b } define <16 x float> @_inreg16xfloat(float %a) { -; CHECK-LABEL: _inreg16xfloat: -; CHECK: ## BB#0: -; CHECK-NEXT: vbroadcastss %xmm0, %zmm0 -; CHECK-NEXT: retq +; ALL-LABEL: _inreg16xfloat: +; ALL: # BB#0: +; ALL-NEXT: vbroadcastss %xmm0, %zmm0 +; ALL-NEXT: retq %b = insertelement <16 x float> undef, float %a, i32 0 %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer ret <16 x float> %c } -;CHECK-LABEL: _ss16xfloat_mask: -;CHECK: vbroadcastss %xmm0, %zmm1 {%k1} -;CHECK: ret define <16 x float> @_ss16xfloat_mask(float %a, <16 x float> %i, <16 x i32> %mask1) { +; ALL-LABEL: _ss16xfloat_mask: +; ALL: # BB#0: +; ALL-NEXT: vpxord %zmm3, %zmm3, %zmm3 +; ALL-NEXT: vpcmpneqd %zmm3, %zmm2, %k1 +; ALL-NEXT: vbroadcastss %xmm0, %zmm1 {%k1} +; ALL-NEXT: vmovaps %zmm1, %zmm0 +; ALL-NEXT: retq %mask = icmp ne <16 x i32> %mask1, zeroinitializer %b = insertelement <16 x float> undef, float %a, i32 0 %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer @@ -49,10 +56,13 @@ define <16 x float> @_ss16xfloat_mask(float %a, <16 x float> %i, <16 x i32> %m ret <16 x float> %r } -;CHECK-LABEL: _ss16xfloat_maskz: -;CHECK: vbroadcastss %xmm0, %zmm0 {%k1} {z} -;CHECK: ret define <16 x float> @_ss16xfloat_maskz(float %a, <16 x i32> %mask1) { +; ALL-LABEL: _ss16xfloat_maskz: +; ALL: # BB#0: +; ALL-NEXT: vpxord %zmm2, %zmm2, %zmm2 +; ALL-NEXT: vpcmpneqd %zmm2, %zmm1, %k1 +; ALL-NEXT: vbroadcastss %xmm0, %zmm0 {%k1} {z} +; ALL-NEXT: retq %mask = icmp ne <16 x i32> %mask1, zeroinitializer %b = insertelement <16 x float> undef, float %a, i32 0 %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer @@ -60,20 +70,24 @@ define <16 x float> @_ss16xfloat_maskz(float %a, <16 x i32> %mask1) { ret <16 x float> %r } -;CHECK-LABEL: _ss16xfloat_load: -;CHECK: vbroadcastss (%{{.*}}, %zmm -;CHECK: ret define <16 x float> @_ss16xfloat_load(float* %a.ptr) { +; ALL-LABEL: _ss16xfloat_load: +; ALL: # BB#0: +; ALL-NEXT: vbroadcastss (%rdi), %zmm0 +; ALL-NEXT: retq %a = load float, float* %a.ptr %b = insertelement <16 x float> undef, float %a, i32 0 %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer ret <16 x float> %c } -;CHECK-LABEL: _ss16xfloat_mask_load: -;CHECK: vbroadcastss (%rdi), %zmm0 {%k1} -;CHECK: ret define <16 x float> @_ss16xfloat_mask_load(float* %a.ptr, <16 x float> %i, <16 x i32> %mask1) { +; ALL-LABEL: _ss16xfloat_mask_load: +; ALL: # BB#0: +; ALL-NEXT: vpxord %zmm2, %zmm2, %zmm2 +; ALL-NEXT: vpcmpneqd %zmm2, %zmm1, %k1 +; ALL-NEXT: vbroadcastss (%rdi), %zmm0 {%k1} +; ALL-NEXT: retq %a = load float, float* %a.ptr %mask = icmp ne <16 x i32> %mask1, zeroinitializer %b = insertelement <16 x float> undef, float %a, i32 0 @@ -82,10 +96,13 @@ define <16 x float> @_ss16xfloat_mask_load(float* %a.ptr, <16 x float> %i, <16 ret <16 x float> %r } -;CHECK-LABEL: _ss16xfloat_maskz_load: -;CHECK: vbroadcastss (%rdi), %zmm0 {%k1} {z} -;CHECK: ret define <16 x float> @_ss16xfloat_maskz_load(float* %a.ptr, <16 x i32> %mask1) { +; ALL-LABEL: _ss16xfloat_maskz_load: +; ALL: # BB#0: +; ALL-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; ALL-NEXT: vpcmpneqd %zmm1, %zmm0, %k1 +; ALL-NEXT: vbroadcastss (%rdi), %zmm0 {%k1} {z} +; ALL-NEXT: retq %a = load float, float* %a.ptr %mask = icmp ne <16 x i32> %mask1, zeroinitializer %b = insertelement <16 x float> undef, float %a, i32 0 @@ -95,19 +112,23 @@ define <16 x float> @_ss16xfloat_maskz_load(float* %a.ptr, <16 x i32> %mask1) } define <8 x double> @_inreg8xdouble(double %a) { -; CHECK-LABEL: _inreg8xdouble: -; CHECK: ## BB#0: -; CHECK-NEXT: vbroadcastsd %xmm0, %zmm0 -; CHECK-NEXT: retq +; ALL-LABEL: _inreg8xdouble: +; ALL: # BB#0: +; ALL-NEXT: vbroadcastsd %xmm0, %zmm0 +; ALL-NEXT: retq %b = insertelement <8 x double> undef, double %a, i32 0 %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer ret <8 x double> %c } -;CHECK-LABEL: _sd8xdouble_mask: -;CHECK: vbroadcastsd %xmm0, %zmm1 {%k1} -;CHECK: ret define <8 x double> @_sd8xdouble_mask(double %a, <8 x double> %i, <8 x i32> %mask1) { +; ALL-LABEL: _sd8xdouble_mask: +; ALL: # BB#0: +; ALL-NEXT: vpxor %ymm3, %ymm3, %ymm3 +; ALL-NEXT: vpcmpneqd %zmm3, %zmm2, %k1 +; ALL-NEXT: vbroadcastsd %xmm0, %zmm1 {%k1} +; ALL-NEXT: vmovaps %zmm1, %zmm0 +; ALL-NEXT: retq %mask = icmp ne <8 x i32> %mask1, zeroinitializer %b = insertelement <8 x double> undef, double %a, i32 0 %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer @@ -115,10 +136,13 @@ define <8 x double> @_sd8xdouble_mask(double %a, <8 x double> %i, <8 x i32> %m ret <8 x double> %r } -;CHECK-LABEL: _sd8xdouble_maskz: -;CHECK: vbroadcastsd %xmm0, %zmm0 {%k1} {z} -;CHECK: ret define <8 x double> @_sd8xdouble_maskz(double %a, <8 x i32> %mask1) { +; ALL-LABEL: _sd8xdouble_maskz: +; ALL: # BB#0: +; ALL-NEXT: vpxor %ymm2, %ymm2, %ymm2 +; ALL-NEXT: vpcmpneqd %zmm2, %zmm1, %k1 +; ALL-NEXT: vbroadcastsd %xmm0, %zmm0 {%k1} {z} +; ALL-NEXT: retq %mask = icmp ne <8 x i32> %mask1, zeroinitializer %b = insertelement <8 x double> undef, double %a, i32 0 %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer @@ -126,20 +150,24 @@ define <8 x double> @_sd8xdouble_maskz(double %a, <8 x i32> %mask1) { ret <8 x double> %r } -;CHECK-LABEL: _sd8xdouble_load: -;CHECK: vbroadcastsd (%rdi), %zmm -;CHECK: ret define <8 x double> @_sd8xdouble_load(double* %a.ptr) { +; ALL-LABEL: _sd8xdouble_load: +; ALL: # BB#0: +; ALL-NEXT: vbroadcastsd (%rdi), %zmm0 +; ALL-NEXT: retq %a = load double, double* %a.ptr %b = insertelement <8 x double> undef, double %a, i32 0 %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer ret <8 x double> %c } -;CHECK-LABEL: _sd8xdouble_mask_load: -;CHECK: vbroadcastsd (%rdi), %zmm0 {%k1} -;CHECK: ret define <8 x double> @_sd8xdouble_mask_load(double* %a.ptr, <8 x double> %i, <8 x i32> %mask1) { +; ALL-LABEL: _sd8xdouble_mask_load: +; ALL: # BB#0: +; ALL-NEXT: vpxor %ymm2, %ymm2, %ymm2 +; ALL-NEXT: vpcmpneqd %zmm2, %zmm1, %k1 +; ALL-NEXT: vbroadcastsd (%rdi), %zmm0 {%k1} +; ALL-NEXT: retq %a = load double, double* %a.ptr %mask = icmp ne <8 x i32> %mask1, zeroinitializer %b = insertelement <8 x double> undef, double %a, i32 0 @@ -149,9 +177,12 @@ define <8 x double> @_sd8xdouble_mask_load(double* %a.ptr, <8 x double> %i, <8 } define <8 x double> @_sd8xdouble_maskz_load(double* %a.ptr, <8 x i32> %mask1) { -; CHECK-LABEL: _sd8xdouble_maskz_load: -; CHECK: vbroadcastsd (%rdi), %zmm0 {%k1} {z} -; CHECK: ret +; ALL-LABEL: _sd8xdouble_maskz_load: +; ALL: # BB#0: +; ALL-NEXT: vpxor %ymm1, %ymm1, %ymm1 +; ALL-NEXT: vpcmpneqd %zmm1, %zmm0, %k1 +; ALL-NEXT: vbroadcastsd (%rdi), %zmm0 {%k1} {z} +; ALL-NEXT: retq %a = load double, double* %a.ptr %mask = icmp ne <8 x i32> %mask1, zeroinitializer %b = insertelement <8 x double> undef, double %a, i32 0 @@ -161,32 +192,32 @@ define <8 x double> @_sd8xdouble_maskz_load(double* %a.ptr, <8 x i32> %mask1) } define <16 x i32> @_xmm16xi32(<16 x i32> %a) { -; CHECK-LABEL: _xmm16xi32: -; CHECK: ## BB#0: -; CHECK-NEXT: vpbroadcastd %xmm0, %zmm0 -; CHECK-NEXT: retq +; ALL-LABEL: _xmm16xi32: +; ALL: # BB#0: +; ALL-NEXT: vpbroadcastd %xmm0, %zmm0 +; ALL-NEXT: retq %b = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32> zeroinitializer ret <16 x i32> %b } define <16 x float> @_xmm16xfloat(<16 x float> %a) { -; CHECK-LABEL: _xmm16xfloat: -; CHECK: ## BB#0: -; CHECK-NEXT: vbroadcastss %xmm0, %zmm0 -; CHECK-NEXT: retq +; ALL-LABEL: _xmm16xfloat: +; ALL: # BB#0: +; ALL-NEXT: vbroadcastss %xmm0, %zmm0 +; ALL-NEXT: retq %b = shufflevector <16 x float> %a, <16 x float> undef, <16 x i32> zeroinitializer ret <16 x float> %b } define <16 x i32> @test_vbroadcast() { -; CHECK-LABEL: test_vbroadcast: -; CHECK: ## BB#0: ## %entry -; CHECK-NEXT: vpxord %zmm0, %zmm0, %zmm0 -; CHECK-NEXT: vcmpunordps %zmm0, %zmm0, %k1 -; CHECK-NEXT: vpbroadcastd {{.*}}(%rip), %zmm0 {%k1} {z} -; CHECK-NEXT: knotw %k1, %k1 -; CHECK-NEXT: vmovdqu32 %zmm0, %zmm0 {%k1} {z} -; CHECK-NEXT: retq +; ALL-LABEL: test_vbroadcast: +; ALL: # BB#0: # %entry +; ALL-NEXT: vpxord %zmm0, %zmm0, %zmm0 +; ALL-NEXT: vcmpunordps %zmm0, %zmm0, %k1 +; ALL-NEXT: vpbroadcastd {{.*}}(%rip), %zmm0 {%k1} {z} +; ALL-NEXT: knotw %k1, %k1 +; ALL-NEXT: vmovdqu32 %zmm0, %zmm0 {%k1} {z} +; ALL-NEXT: retq entry: %0 = sext <16 x i1> zeroinitializer to <16 x i32> %1 = fcmp uno <16 x float> undef, zeroinitializer @@ -198,10 +229,10 @@ entry: ; We implement the set1 intrinsics with vector initializers. Verify that the ; IR generated will produce broadcasts at the end. define <8 x double> @test_set1_pd(double %d) #2 { -; CHECK-LABEL: test_set1_pd: -; CHECK: ## BB#0: ## %entry -; CHECK-NEXT: vbroadcastsd %xmm0, %zmm0 -; CHECK-NEXT: retq +; ALL-LABEL: test_set1_pd: +; ALL: # BB#0: # %entry +; ALL-NEXT: vbroadcastsd %xmm0, %zmm0 +; ALL-NEXT: retq entry: %vecinit.i = insertelement <8 x double> undef, double %d, i32 0 %vecinit1.i = insertelement <8 x double> %vecinit.i, double %d, i32 1 @@ -215,10 +246,10 @@ entry: } define <8 x i64> @test_set1_epi64(i64 %d) #2 { -; CHECK-LABEL: test_set1_epi64: -; CHECK: ## BB#0: ## %entry -; CHECK-NEXT: vpbroadcastq %rdi, %zmm0 -; CHECK-NEXT: retq +; ALL-LABEL: test_set1_epi64: +; ALL: # BB#0: # %entry +; ALL-NEXT: vpbroadcastq %rdi, %zmm0 +; ALL-NEXT: retq entry: %vecinit.i = insertelement <8 x i64> undef, i64 %d, i32 0 %vecinit1.i = insertelement <8 x i64> %vecinit.i, i64 %d, i32 1 @@ -232,10 +263,10 @@ entry: } define <16 x float> @test_set1_ps(float %f) #2 { -; CHECK-LABEL: test_set1_ps: -; CHECK: ## BB#0: ## %entry -; CHECK-NEXT: vbroadcastss %xmm0, %zmm0 -; CHECK-NEXT: retq +; ALL-LABEL: test_set1_ps: +; ALL: # BB#0: # %entry +; ALL-NEXT: vbroadcastss %xmm0, %zmm0 +; ALL-NEXT: retq entry: %vecinit.i = insertelement <16 x float> undef, float %f, i32 0 %vecinit1.i = insertelement <16 x float> %vecinit.i, float %f, i32 1 @@ -257,10 +288,10 @@ entry: } define <16 x i32> @test_set1_epi32(i32 %f) #2 { -; CHECK-LABEL: test_set1_epi32: -; CHECK: ## BB#0: ## %entry -; CHECK-NEXT: vpbroadcastd %edi, %zmm0 -; CHECK-NEXT: retq +; ALL-LABEL: test_set1_epi32: +; ALL: # BB#0: # %entry +; ALL-NEXT: vpbroadcastd %edi, %zmm0 +; ALL-NEXT: retq entry: %vecinit.i = insertelement <16 x i32> undef, i32 %f, i32 0 %vecinit1.i = insertelement <16 x i32> %vecinit.i, i32 %f, i32 1 @@ -284,10 +315,10 @@ entry: ; We implement the scalar broadcast intrinsics with vector initializers. ; Verify that the IR generated will produce the broadcast at the end. define <8 x double> @test_mm512_broadcastsd_pd(<2 x double> %a) { -; CHECK-LABEL: test_mm512_broadcastsd_pd: -; CHECK: ## BB#0: ## %entry -; CHECK-NEXT: vbroadcastsd %xmm0, %zmm0 -; CHECK-NEXT: retq +; ALL-LABEL: test_mm512_broadcastsd_pd: +; ALL: # BB#0: # %entry +; ALL-NEXT: vbroadcastsd %xmm0, %zmm0 +; ALL-NEXT: retq entry: %0 = extractelement <2 x double> %a, i32 0 %vecinit.i = insertelement <8 x double> undef, double %0, i32 0 @@ -301,30 +332,69 @@ entry: ret <8 x double> %vecinit7.i } -; CHECK-LABEL: test1 -; CHECK: vbroadcastss define <16 x float> @test1(<8 x float>%a) { +; ALL-LABEL: test1: +; ALL: # BB#0: +; ALL-NEXT: vbroadcastss %xmm0, %zmm0 +; ALL-NEXT: retq %res = shufflevector <8 x float> %a, <8 x float> undef, <16 x i32> zeroinitializer ret <16 x float>%res } -; CHECK-LABEL: test2 -; CHECK: vbroadcastsd define <8 x double> @test2(<4 x double>%a) { +; ALL-LABEL: test2: +; ALL: # BB#0: +; ALL-NEXT: vbroadcastsd %xmm0, %zmm0 +; ALL-NEXT: retq %res = shufflevector <4 x double> %a, <4 x double> undef, <8 x i32> zeroinitializer ret <8 x double>%res } -; CHECK-LABEL: test3 -; CHECK: vpbroadcastd -define <16 x i32> @test3(<8 x i32>%a) { +define <64 x i8> @_invec32xi8(<32 x i8>%a) { +; AVX512F-LABEL: _invec32xi8: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm0 +; AVX512F-NEXT: vmovaps %zmm0, %zmm1 +; AVX512F-NEXT: retq +; +; AVX512BW-LABEL: _invec32xi8: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vpbroadcastb %xmm0, %zmm0 +; AVX512BW-NEXT: retq + %res = shufflevector <32 x i8> %a, <32 x i8> undef, <64 x i32> zeroinitializer + ret <64 x i8>%res +} + +define <32 x i16> @_invec16xi16(<16 x i16>%a) { +; AVX512F-LABEL: _invec16xi16: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpbroadcastw %xmm0, %ymm0 +; AVX512F-NEXT: vmovaps %zmm0, %zmm1 +; AVX512F-NEXT: retq +; +; AVX512BW-LABEL: _invec16xi16: +; AVX512BW: # BB#0: +; AVX512BW-NEXT: vpbroadcastw %xmm0, %zmm0 +; AVX512BW-NEXT: retq + %res = shufflevector <16 x i16> %a, <16 x i16> undef, <32 x i32> zeroinitializer + ret <32 x i16>%res +} + +define <16 x i32> @_invec8xi32(<8 x i32>%a) { +; ALL-LABEL: _invec8xi32: +; ALL: # BB#0: +; ALL-NEXT: vpbroadcastd %xmm0, %zmm0 +; ALL-NEXT: retq %res = shufflevector <8 x i32> %a, <8 x i32> undef, <16 x i32> zeroinitializer ret <16 x i32>%res } -; CHECK-LABEL: test4 -; CHECK: vpbroadcastq -define <8 x i64> @test4(<4 x i64>%a) { +define <8 x i64> @_invec4xi64(<4 x i64>%a) { +; ALL-LABEL: _invec4xi64: +; ALL: # BB#0: +; ALL-NEXT: vpbroadcastq %xmm0, %zmm0 +; ALL-NEXT: retq %res = shufflevector <4 x i64> %a, <4 x i64> undef, <8 x i32> zeroinitializer ret <8 x i64>%res } + diff --git a/test/CodeGen/X86/avx512-vec-cmp.ll b/test/CodeGen/X86/avx512-vec-cmp.ll index 6a4a3aa7e371..a8c558df9de8 100644 --- a/test/CodeGen/X86/avx512-vec-cmp.ll +++ b/test/CodeGen/X86/avx512-vec-cmp.ll @@ -152,7 +152,6 @@ define <8 x i32> @test11_unsigned(<8 x i32> %x, <8 x i32> %y) nounwind { ret <8 x i32> %max } - define i16 @test12(<16 x i64> %a, <16 x i64> %b) nounwind { ; KNL-LABEL: test12: ; KNL: ## BB#0: @@ -166,6 +165,32 @@ define i16 @test12(<16 x i64> %a, <16 x i64> %b) nounwind { ret i16 %res1 } +define i32 @test12_v32i32(<32 x i32> %a, <32 x i32> %b) nounwind { +; SKX-LABEL: test12_v32i32: +; SKX: ## BB#0: +; SKX-NEXT: vpcmpeqd %zmm2, %zmm0, %k0 +; SKX-NEXT: vpcmpeqd %zmm3, %zmm1, %k1 +; SKX-NEXT: kunpckwd %k0, %k1, %k0 +; SKX-NEXT: kmovd %k0, %eax +; SKX-NEXT: retq + %res = icmp eq <32 x i32> %a, %b + %res1 = bitcast <32 x i1> %res to i32 + ret i32 %res1 +} + +define i64 @test12_v64i16(<64 x i16> %a, <64 x i16> %b) nounwind { +; SKX-LABEL: test12_v64i16: +; SKX: ## BB#0: +; SKX-NEXT: vpcmpeqw %zmm2, %zmm0, %k0 +; SKX-NEXT: vpcmpeqw %zmm3, %zmm1, %k1 +; SKX-NEXT: kunpckdq %k0, %k1, %k0 +; SKX-NEXT: kmovq %k0, %rax +; SKX-NEXT: retq + %res = icmp eq <64 x i16> %a, %b + %res1 = bitcast <64 x i1> %res to i64 + ret i64 %res1 +} + define <16 x i32> @test13(<16 x float>%a, <16 x float>%b) ; KNL-LABEL: test13: ; KNL: ## BB#0: diff --git a/test/CodeGen/X86/avx512bw-intrinsics.ll b/test/CodeGen/X86/avx512bw-intrinsics.ll index 71bf63ed44d0..5f3d16d4efbb 100644 --- a/test/CodeGen/X86/avx512bw-intrinsics.ll +++ b/test/CodeGen/X86/avx512bw-intrinsics.ll @@ -1,15 +1,51 @@ -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx --show-mc-encoding| FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512BW +; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mcpu=knl -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512F-32 define i64 @test_pcmpeq_b(<64 x i8> %a, <64 x i8> %b) { -; CHECK-LABEL: test_pcmpeq_b -; CHECK: vpcmpeqb %zmm1, %zmm0, %k0 ## +; AVX512BW-LABEL: test_pcmpeq_b: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpcmpeqb %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_pcmpeq_b: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: subl $12, %esp +; AVX512F-32-NEXT: .Ltmp0: +; AVX512F-32-NEXT: .cfi_def_cfa_offset 16 +; AVX512F-32-NEXT: vpcmpeqb %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, (%esp) +; AVX512F-32-NEXT: movl (%esp), %eax +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: addl $12, %esp +; AVX512F-32-NEXT: retl %res = call i64 @llvm.x86.avx512.mask.pcmpeq.b.512(<64 x i8> %a, <64 x i8> %b, i64 -1) ret i64 %res } define i64 @test_mask_pcmpeq_b(<64 x i8> %a, <64 x i8> %b, i64 %mask) { -; CHECK-LABEL: test_mask_pcmpeq_b -; CHECK: vpcmpeqb %zmm1, %zmm0, %k0 {%k1} ## +; AVX512BW-LABEL: test_mask_pcmpeq_b: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpcmpeqb %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_pcmpeq_b: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: subl $12, %esp +; AVX512F-32-NEXT: .Ltmp1: +; AVX512F-32-NEXT: .cfi_def_cfa_offset 16 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpcmpeqb %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, (%esp) +; AVX512F-32-NEXT: movl (%esp), %eax +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: addl $12, %esp +; AVX512F-32-NEXT: retl %res = call i64 @llvm.x86.avx512.mask.pcmpeq.b.512(<64 x i8> %a, <64 x i8> %b, i64 %mask) ret i64 %res } @@ -17,15 +53,35 @@ define i64 @test_mask_pcmpeq_b(<64 x i8> %a, <64 x i8> %b, i64 %mask) { declare i64 @llvm.x86.avx512.mask.pcmpeq.b.512(<64 x i8>, <64 x i8>, i64) define i32 @test_pcmpeq_w(<32 x i16> %a, <32 x i16> %b) { -; CHECK-LABEL: test_pcmpeq_w -; CHECK: vpcmpeqw %zmm1, %zmm0, %k0 ## +; AVX512BW-LABEL: test_pcmpeq_w: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpcmpeqw %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_pcmpeq_w: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpcmpeqw %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: retl %res = call i32 @llvm.x86.avx512.mask.pcmpeq.w.512(<32 x i16> %a, <32 x i16> %b, i32 -1) ret i32 %res } define i32 @test_mask_pcmpeq_w(<32 x i16> %a, <32 x i16> %b, i32 %mask) { -; CHECK-LABEL: test_mask_pcmpeq_w -; CHECK: vpcmpeqw %zmm1, %zmm0, %k0 {%k1} ## +; AVX512BW-LABEL: test_mask_pcmpeq_w: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpcmpeqw %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_pcmpeq_w: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpcmpeqw %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: retl %res = call i32 @llvm.x86.avx512.mask.pcmpeq.w.512(<32 x i16> %a, <32 x i16> %b, i32 %mask) ret i32 %res } @@ -33,15 +89,49 @@ define i32 @test_mask_pcmpeq_w(<32 x i16> %a, <32 x i16> %b, i32 %mask) { declare i32 @llvm.x86.avx512.mask.pcmpeq.w.512(<32 x i16>, <32 x i16>, i32) define i64 @test_pcmpgt_b(<64 x i8> %a, <64 x i8> %b) { -; CHECK-LABEL: test_pcmpgt_b -; CHECK: vpcmpgtb %zmm1, %zmm0, %k0 ## +; AVX512BW-LABEL: test_pcmpgt_b: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpcmpgtb %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_pcmpgt_b: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: subl $12, %esp +; AVX512F-32-NEXT: .Ltmp2: +; AVX512F-32-NEXT: .cfi_def_cfa_offset 16 +; AVX512F-32-NEXT: vpcmpgtb %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, (%esp) +; AVX512F-32-NEXT: movl (%esp), %eax +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: addl $12, %esp +; AVX512F-32-NEXT: retl %res = call i64 @llvm.x86.avx512.mask.pcmpgt.b.512(<64 x i8> %a, <64 x i8> %b, i64 -1) ret i64 %res } define i64 @test_mask_pcmpgt_b(<64 x i8> %a, <64 x i8> %b, i64 %mask) { -; CHECK-LABEL: test_mask_pcmpgt_b -; CHECK: vpcmpgtb %zmm1, %zmm0, %k0 {%k1} ## +; AVX512BW-LABEL: test_mask_pcmpgt_b: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpcmpgtb %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_pcmpgt_b: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: subl $12, %esp +; AVX512F-32-NEXT: .Ltmp3: +; AVX512F-32-NEXT: .cfi_def_cfa_offset 16 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpcmpgtb %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, (%esp) +; AVX512F-32-NEXT: movl (%esp), %eax +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: addl $12, %esp +; AVX512F-32-NEXT: retl %res = call i64 @llvm.x86.avx512.mask.pcmpgt.b.512(<64 x i8> %a, <64 x i8> %b, i64 %mask) ret i64 %res } @@ -49,357 +139,839 @@ define i64 @test_mask_pcmpgt_b(<64 x i8> %a, <64 x i8> %b, i64 %mask) { declare i64 @llvm.x86.avx512.mask.pcmpgt.b.512(<64 x i8>, <64 x i8>, i64) define i32 @test_pcmpgt_w(<32 x i16> %a, <32 x i16> %b) { -; CHECK-LABEL: test_pcmpgt_w -; CHECK: vpcmpgtw %zmm1, %zmm0, %k0 ## +; AVX512BW-LABEL: test_pcmpgt_w: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpcmpgtw %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_pcmpgt_w: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpcmpgtw %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: retl %res = call i32 @llvm.x86.avx512.mask.pcmpgt.w.512(<32 x i16> %a, <32 x i16> %b, i32 -1) ret i32 %res } define i32 @test_mask_pcmpgt_w(<32 x i16> %a, <32 x i16> %b, i32 %mask) { -; CHECK-LABEL: test_mask_pcmpgt_w -; CHECK: vpcmpgtw %zmm1, %zmm0, %k0 {%k1} ## +; AVX512BW-LABEL: test_mask_pcmpgt_w: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpcmpgtw %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_pcmpgt_w: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpcmpgtw %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: retl %res = call i32 @llvm.x86.avx512.mask.pcmpgt.w.512(<32 x i16> %a, <32 x i16> %b, i32 %mask) ret i32 %res } declare i32 @llvm.x86.avx512.mask.pcmpgt.w.512(<32 x i16>, <32 x i16>, i32) -define <8 x i64> @test_cmp_b_512(<64 x i8> %a0, <64 x i8> %a1) { -; CHECK_LABEL: test_cmp_b_512 -; CHECK: vpcmpeqb %zmm1, %zmm0, %k0 ## +define i64 @test_cmp_b_512(<64 x i8> %a0, <64 x i8> %a1) { +; AVX512BW-LABEL: test_cmp_b_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpcmpeqb %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: vpcmpltb %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rcx +; AVX512BW-NEXT: addq %rax, %rcx +; AVX512BW-NEXT: vpcmpleb %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: addq %rcx, %rax +; AVX512BW-NEXT: vpcmpunordb %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rcx +; AVX512BW-NEXT: addq %rax, %rcx +; AVX512BW-NEXT: vpcmpneqb %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: addq %rcx, %rax +; AVX512BW-NEXT: vpcmpnltb %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rcx +; AVX512BW-NEXT: addq %rax, %rcx +; AVX512BW-NEXT: vpcmpnleb %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rdx +; AVX512BW-NEXT: addq %rcx, %rdx +; AVX512BW-NEXT: vpcmpordb %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: addq %rdx, %rax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_cmp_b_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: subl $68, %esp +; AVX512F-32-NEXT: .Ltmp4: +; AVX512F-32-NEXT: .cfi_def_cfa_offset 72 +; AVX512F-32-NEXT: vpcmpeqb %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpltb %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpleb %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpunordb %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpneqb %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpnltb %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpnleb %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpordb %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, (%esp) +; AVX512F-32-NEXT: addl (%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: addl $68, %esp +; AVX512F-32-NEXT: retl %res0 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 0, i64 -1) - %vec0 = insertelement <8 x i64> undef, i64 %res0, i32 0 -; CHECK: vpcmpltb %zmm1, %zmm0, %k0 ## %res1 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 1, i64 -1) - %vec1 = insertelement <8 x i64> %vec0, i64 %res1, i32 1 -; CHECK: vpcmpleb %zmm1, %zmm0, %k0 ## + %ret1 = add i64 %res0, %res1 %res2 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 2, i64 -1) - %vec2 = insertelement <8 x i64> %vec1, i64 %res2, i32 2 -; CHECK: vpcmpunordb %zmm1, %zmm0, %k0 ## + %ret2 = add i64 %ret1, %res2 %res3 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 3, i64 -1) - %vec3 = insertelement <8 x i64> %vec2, i64 %res3, i32 3 -; CHECK: vpcmpneqb %zmm1, %zmm0, %k0 ## + %ret3 = add i64 %ret2, %res3 %res4 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 4, i64 -1) - %vec4 = insertelement <8 x i64> %vec3, i64 %res4, i32 4 -; CHECK: vpcmpnltb %zmm1, %zmm0, %k0 ## + %ret4 = add i64 %ret3, %res4 %res5 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 5, i64 -1) - %vec5 = insertelement <8 x i64> %vec4, i64 %res5, i32 5 -; CHECK: vpcmpnleb %zmm1, %zmm0, %k0 ## + %ret5 = add i64 %ret4, %res5 %res6 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 6, i64 -1) - %vec6 = insertelement <8 x i64> %vec5, i64 %res6, i32 6 -; CHECK: vpcmpordb %zmm1, %zmm0, %k0 ## + %ret6 = add i64 %ret5, %res6 %res7 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 7, i64 -1) - %vec7 = insertelement <8 x i64> %vec6, i64 %res7, i32 7 - ret <8 x i64> %vec7 -} - -define <8 x i64> @test_mask_cmp_b_512(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) { -; CHECK_LABEL: test_mask_cmp_b_512 -; CHECK: vpcmpeqb %zmm1, %zmm0, %k0 {%k1} ## + %ret7 = add i64 %ret6, %res7 + ret i64 %ret7 +} + +define i64 @test_mask_cmp_b_512(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) { +; AVX512BW-LABEL: test_mask_cmp_b_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpcmpeqb %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: vpcmpltb %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rcx +; AVX512BW-NEXT: addq %rax, %rcx +; AVX512BW-NEXT: vpcmpleb %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: addq %rcx, %rax +; AVX512BW-NEXT: vpcmpunordb %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rcx +; AVX512BW-NEXT: addq %rax, %rcx +; AVX512BW-NEXT: vpcmpneqb %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: addq %rcx, %rax +; AVX512BW-NEXT: vpcmpnltb %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rcx +; AVX512BW-NEXT: addq %rax, %rcx +; AVX512BW-NEXT: vpcmpnleb %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rdx +; AVX512BW-NEXT: addq %rcx, %rdx +; AVX512BW-NEXT: vpcmpordb %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: addq %rdx, %rax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_cmp_b_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: subl $68, %esp +; AVX512F-32-NEXT: .Ltmp5: +; AVX512F-32-NEXT: .cfi_def_cfa_offset 72 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpcmpeqb %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, (%esp) +; AVX512F-32-NEXT: movl (%esp), %eax +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpltb %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpleb %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpunordb %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpneqb %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpnltb %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpnleb %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpordb %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: addl $68, %esp +; AVX512F-32-NEXT: retl %res0 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 0, i64 %mask) - %vec0 = insertelement <8 x i64> undef, i64 %res0, i32 0 -; CHECK: vpcmpltb %zmm1, %zmm0, %k0 {%k1} ## %res1 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 1, i64 %mask) - %vec1 = insertelement <8 x i64> %vec0, i64 %res1, i32 1 -; CHECK: vpcmpleb %zmm1, %zmm0, %k0 {%k1} ## + %ret1 = add i64 %res0, %res1 %res2 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 2, i64 %mask) - %vec2 = insertelement <8 x i64> %vec1, i64 %res2, i32 2 -; CHECK: vpcmpunordb %zmm1, %zmm0, %k0 {%k1} ## + %ret2 = add i64 %ret1, %res2 %res3 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 3, i64 %mask) - %vec3 = insertelement <8 x i64> %vec2, i64 %res3, i32 3 -; CHECK: vpcmpneqb %zmm1, %zmm0, %k0 {%k1} ## + %ret3 = add i64 %ret2, %res3 %res4 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 4, i64 %mask) - %vec4 = insertelement <8 x i64> %vec3, i64 %res4, i32 4 -; CHECK: vpcmpnltb %zmm1, %zmm0, %k0 {%k1} ## + %ret4 = add i64 %ret3, %res4 %res5 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 5, i64 %mask) - %vec5 = insertelement <8 x i64> %vec4, i64 %res5, i32 5 -; CHECK: vpcmpnleb %zmm1, %zmm0, %k0 {%k1} ## + %ret5 = add i64 %ret4, %res5 %res6 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 6, i64 %mask) - %vec6 = insertelement <8 x i64> %vec5, i64 %res6, i32 6 -; CHECK: vpcmpordb %zmm1, %zmm0, %k0 {%k1} ## + %ret6 = add i64 %ret5, %res6 %res7 = call i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 7, i64 %mask) - %vec7 = insertelement <8 x i64> %vec6, i64 %res7, i32 7 - ret <8 x i64> %vec7 + %ret7 = add i64 %ret6, %res7 + ret i64 %ret7 } declare i64 @llvm.x86.avx512.mask.cmp.b.512(<64 x i8>, <64 x i8>, i32, i64) nounwind readnone -define <8 x i64> @test_ucmp_b_512(<64 x i8> %a0, <64 x i8> %a1) { -; CHECK_LABEL: test_ucmp_b_512 -; CHECK: vpcmpequb %zmm1, %zmm0, %k0 ## +define i64 @test_ucmp_b_512(<64 x i8> %a0, <64 x i8> %a1) { +; AVX512BW-LABEL: test_ucmp_b_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpcmpequb %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: vpcmpltub %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rcx +; AVX512BW-NEXT: addq %rax, %rcx +; AVX512BW-NEXT: vpcmpleub %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: addq %rcx, %rax +; AVX512BW-NEXT: vpcmpunordub %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rcx +; AVX512BW-NEXT: addq %rax, %rcx +; AVX512BW-NEXT: vpcmpnequb %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: addq %rcx, %rax +; AVX512BW-NEXT: vpcmpnltub %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rcx +; AVX512BW-NEXT: addq %rax, %rcx +; AVX512BW-NEXT: vpcmpnleub %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rdx +; AVX512BW-NEXT: addq %rcx, %rdx +; AVX512BW-NEXT: vpcmpordub %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: addq %rdx, %rax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_ucmp_b_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: subl $68, %esp +; AVX512F-32-NEXT: .Ltmp6: +; AVX512F-32-NEXT: .cfi_def_cfa_offset 72 +; AVX512F-32-NEXT: vpcmpequb %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpltub %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpleub %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpunordub %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpnequb %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpnltub %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpnleub %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpordub %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, (%esp) +; AVX512F-32-NEXT: addl (%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: addl $68, %esp +; AVX512F-32-NEXT: retl %res0 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 0, i64 -1) - %vec0 = insertelement <8 x i64> undef, i64 %res0, i32 0 -; CHECK: vpcmpltub %zmm1, %zmm0, %k0 ## %res1 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 1, i64 -1) - %vec1 = insertelement <8 x i64> %vec0, i64 %res1, i32 1 -; CHECK: vpcmpleub %zmm1, %zmm0, %k0 ## + %ret1 = add i64 %res0, %res1 %res2 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 2, i64 -1) - %vec2 = insertelement <8 x i64> %vec1, i64 %res2, i32 2 -; CHECK: vpcmpunordub %zmm1, %zmm0, %k0 ## + %ret2 = add i64 %ret1, %res2 %res3 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 3, i64 -1) - %vec3 = insertelement <8 x i64> %vec2, i64 %res3, i32 3 -; CHECK: vpcmpnequb %zmm1, %zmm0, %k0 ## + %ret3 = add i64 %ret2, %res3 %res4 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 4, i64 -1) - %vec4 = insertelement <8 x i64> %vec3, i64 %res4, i32 4 -; CHECK: vpcmpnltub %zmm1, %zmm0, %k0 ## + %ret4 = add i64 %ret3, %res4 %res5 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 5, i64 -1) - %vec5 = insertelement <8 x i64> %vec4, i64 %res5, i32 5 -; CHECK: vpcmpnleub %zmm1, %zmm0, %k0 ## + %ret5 = add i64 %ret4, %res5 %res6 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 6, i64 -1) - %vec6 = insertelement <8 x i64> %vec5, i64 %res6, i32 6 -; CHECK: vpcmpordub %zmm1, %zmm0, %k0 ## + %ret6 = add i64 %ret5, %res6 %res7 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 7, i64 -1) - %vec7 = insertelement <8 x i64> %vec6, i64 %res7, i32 7 - ret <8 x i64> %vec7 -} - -define <8 x i64> @test_mask_x86_avx512_ucmp_b_512(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) { -; CHECK_LABEL: test_mask_ucmp_b_512 -; CHECK: vpcmpequb %zmm1, %zmm0, %k0 {%k1} ## + %ret7 = add i64 %ret6, %res7 + ret i64 %ret7 +} + +define i64 @test_mask_x86_avx512_ucmp_b_512(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) { +; AVX512BW-LABEL: test_mask_x86_avx512_ucmp_b_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpcmpequb %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: vpcmpltub %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rcx +; AVX512BW-NEXT: addq %rax, %rcx +; AVX512BW-NEXT: vpcmpleub %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: addq %rcx, %rax +; AVX512BW-NEXT: vpcmpunordub %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rcx +; AVX512BW-NEXT: addq %rax, %rcx +; AVX512BW-NEXT: vpcmpnequb %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: addq %rcx, %rax +; AVX512BW-NEXT: vpcmpnltub %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rcx +; AVX512BW-NEXT: addq %rax, %rcx +; AVX512BW-NEXT: vpcmpnleub %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rdx +; AVX512BW-NEXT: addq %rcx, %rdx +; AVX512BW-NEXT: vpcmpordub %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: addq %rdx, %rax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_x86_avx512_ucmp_b_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: subl $68, %esp +; AVX512F-32-NEXT: .Ltmp7: +; AVX512F-32-NEXT: .cfi_def_cfa_offset 72 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpcmpequb %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, (%esp) +; AVX512F-32-NEXT: movl (%esp), %eax +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpltub %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpleub %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpunordub %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpnequb %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpnltub %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpnleub %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: vpcmpordub %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovq %k0, {{[0-9]+}}(%esp) +; AVX512F-32-NEXT: addl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: adcl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: addl $68, %esp +; AVX512F-32-NEXT: retl %res0 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 0, i64 %mask) - %vec0 = insertelement <8 x i64> undef, i64 %res0, i32 0 -; CHECK: vpcmpltub %zmm1, %zmm0, %k0 {%k1} ## %res1 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 1, i64 %mask) - %vec1 = insertelement <8 x i64> %vec0, i64 %res1, i32 1 -; CHECK: vpcmpleub %zmm1, %zmm0, %k0 {%k1} ## + %ret1 = add i64 %res0, %res1 %res2 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 2, i64 %mask) - %vec2 = insertelement <8 x i64> %vec1, i64 %res2, i32 2 -; CHECK: vpcmpunordub %zmm1, %zmm0, %k0 {%k1} ## + %ret2 = add i64 %ret1, %res2 %res3 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 3, i64 %mask) - %vec3 = insertelement <8 x i64> %vec2, i64 %res3, i32 3 -; CHECK: vpcmpnequb %zmm1, %zmm0, %k0 {%k1} ## + %ret3 = add i64 %ret2, %res3 %res4 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 4, i64 %mask) - %vec4 = insertelement <8 x i64> %vec3, i64 %res4, i32 4 -; CHECK: vpcmpnltub %zmm1, %zmm0, %k0 {%k1} ## + %ret4 = add i64 %ret3, %res4 %res5 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 5, i64 %mask) - %vec5 = insertelement <8 x i64> %vec4, i64 %res5, i32 5 -; CHECK: vpcmpnleub %zmm1, %zmm0, %k0 {%k1} ## + %ret5 = add i64 %ret4, %res5 %res6 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 6, i64 %mask) - %vec6 = insertelement <8 x i64> %vec5, i64 %res6, i32 6 -; CHECK: vpcmpordub %zmm1, %zmm0, %k0 {%k1} ## + %ret6 = add i64 %ret5, %res6 %res7 = call i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8> %a0, <64 x i8> %a1, i32 7, i64 %mask) - %vec7 = insertelement <8 x i64> %vec6, i64 %res7, i32 7 - ret <8 x i64> %vec7 + %ret7 = add i64 %ret6, %res7 + ret i64 %ret7 } declare i64 @llvm.x86.avx512.mask.ucmp.b.512(<64 x i8>, <64 x i8>, i32, i64) nounwind readnone -define <8 x i32> @test_cmp_w_512(<32 x i16> %a0, <32 x i16> %a1) { -; CHECK_LABEL: test_cmp_w_512 -; CHECK: vpcmpeqw %zmm1, %zmm0, %k0 ## +define i32 @test_cmp_w_512(<32 x i16> %a0, <32 x i16> %a1) { +; AVX512BW-LABEL: test_cmp_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpcmpeqw %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: vpcmpltw %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %ecx +; AVX512BW-NEXT: addl %eax, %ecx +; AVX512BW-NEXT: vpcmplew %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: addl %ecx, %eax +; AVX512BW-NEXT: vpcmpunordw %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %ecx +; AVX512BW-NEXT: addl %eax, %ecx +; AVX512BW-NEXT: vpcmpneqw %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: addl %ecx, %eax +; AVX512BW-NEXT: vpcmpnltw %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %ecx +; AVX512BW-NEXT: addl %eax, %ecx +; AVX512BW-NEXT: vpcmpnlew %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %edx +; AVX512BW-NEXT: addl %ecx, %edx +; AVX512BW-NEXT: vpcmpordw %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: addl %edx, %eax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_cmp_w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpcmpeqw %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: vpcmpltw %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %ecx +; AVX512F-32-NEXT: addl %eax, %ecx +; AVX512F-32-NEXT: vpcmplew %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: addl %ecx, %eax +; AVX512F-32-NEXT: vpcmpunordw %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %ecx +; AVX512F-32-NEXT: addl %eax, %ecx +; AVX512F-32-NEXT: vpcmpneqw %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: addl %ecx, %eax +; AVX512F-32-NEXT: vpcmpnltw %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %ecx +; AVX512F-32-NEXT: addl %eax, %ecx +; AVX512F-32-NEXT: vpcmpnlew %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %edx +; AVX512F-32-NEXT: addl %ecx, %edx +; AVX512F-32-NEXT: vpcmpordw %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: addl %edx, %eax +; AVX512F-32-NEXT: retl %res0 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 0, i32 -1) - %vec0 = insertelement <8 x i32> undef, i32 %res0, i32 0 -; CHECK: vpcmpltw %zmm1, %zmm0, %k0 ## %res1 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 1, i32 -1) - %vec1 = insertelement <8 x i32> %vec0, i32 %res1, i32 1 -; CHECK: vpcmplew %zmm1, %zmm0, %k0 ## + %ret1 = add i32 %res0, %res1 %res2 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 2, i32 -1) - %vec2 = insertelement <8 x i32> %vec1, i32 %res2, i32 2 -; CHECK: vpcmpunordw %zmm1, %zmm0, %k0 ## + %ret2 = add i32 %ret1, %res2 %res3 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 3, i32 -1) - %vec3 = insertelement <8 x i32> %vec2, i32 %res3, i32 3 -; CHECK: vpcmpneqw %zmm1, %zmm0, %k0 ## + %ret3 = add i32 %ret2, %res3 %res4 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 4, i32 -1) - %vec4 = insertelement <8 x i32> %vec3, i32 %res4, i32 4 -; CHECK: vpcmpnltw %zmm1, %zmm0, %k0 ## + %ret4 = add i32 %ret3, %res4 %res5 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 5, i32 -1) - %vec5 = insertelement <8 x i32> %vec4, i32 %res5, i32 5 -; CHECK: vpcmpnlew %zmm1, %zmm0, %k0 ## + %ret5 = add i32 %ret4, %res5 %res6 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 6, i32 -1) - %vec6 = insertelement <8 x i32> %vec5, i32 %res6, i32 6 -; CHECK: vpcmpordw %zmm1, %zmm0, %k0 ## + %ret6 = add i32 %ret5, %res6 %res7 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 7, i32 -1) - %vec7 = insertelement <8 x i32> %vec6, i32 %res7, i32 7 - ret <8 x i32> %vec7 -} - -define <8 x i32> @test_mask_cmp_w_512(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) { -; CHECK_LABEL: test_mask_cmp_w_512 -; CHECK: vpcmpeqw %zmm1, %zmm0, %k0 {%k1} ## + %ret7 = add i32 %ret6, %res7 + ret i32 %ret7 +} + +define i32 @test_mask_cmp_w_512(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) { +; AVX512BW-LABEL: test_mask_cmp_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpcmpeqw %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: vpcmpltw %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %ecx +; AVX512BW-NEXT: addl %eax, %ecx +; AVX512BW-NEXT: vpcmplew %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: addl %ecx, %eax +; AVX512BW-NEXT: vpcmpunordw %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %ecx +; AVX512BW-NEXT: addl %eax, %ecx +; AVX512BW-NEXT: vpcmpneqw %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: addl %ecx, %eax +; AVX512BW-NEXT: vpcmpnltw %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %ecx +; AVX512BW-NEXT: addl %eax, %ecx +; AVX512BW-NEXT: vpcmpnlew %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %edx +; AVX512BW-NEXT: addl %ecx, %edx +; AVX512BW-NEXT: vpcmpordw %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: addl %edx, %eax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_cmp_w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpcmpeqw %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: vpcmpltw %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %ecx +; AVX512F-32-NEXT: addl %eax, %ecx +; AVX512F-32-NEXT: vpcmplew %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: addl %ecx, %eax +; AVX512F-32-NEXT: vpcmpunordw %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %ecx +; AVX512F-32-NEXT: addl %eax, %ecx +; AVX512F-32-NEXT: vpcmpneqw %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: addl %ecx, %eax +; AVX512F-32-NEXT: vpcmpnltw %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %ecx +; AVX512F-32-NEXT: addl %eax, %ecx +; AVX512F-32-NEXT: vpcmpnlew %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %edx +; AVX512F-32-NEXT: addl %ecx, %edx +; AVX512F-32-NEXT: vpcmpordw %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: addl %edx, %eax +; AVX512F-32-NEXT: retl %res0 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 0, i32 %mask) - %vec0 = insertelement <8 x i32> undef, i32 %res0, i32 0 -; CHECK: vpcmpltw %zmm1, %zmm0, %k0 {%k1} ## %res1 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 1, i32 %mask) - %vec1 = insertelement <8 x i32> %vec0, i32 %res1, i32 1 -; CHECK: vpcmplew %zmm1, %zmm0, %k0 {%k1} ## + %ret1 = add i32 %res0, %res1 %res2 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 2, i32 %mask) - %vec2 = insertelement <8 x i32> %vec1, i32 %res2, i32 2 -; CHECK: vpcmpunordw %zmm1, %zmm0, %k0 {%k1} ## + %ret2 = add i32 %ret1, %res2 %res3 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 3, i32 %mask) - %vec3 = insertelement <8 x i32> %vec2, i32 %res3, i32 3 -; CHECK: vpcmpneqw %zmm1, %zmm0, %k0 {%k1} ## + %ret3 = add i32 %ret2, %res3 %res4 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 4, i32 %mask) - %vec4 = insertelement <8 x i32> %vec3, i32 %res4, i32 4 -; CHECK: vpcmpnltw %zmm1, %zmm0, %k0 {%k1} ## + %ret4 = add i32 %ret3, %res4 %res5 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 5, i32 %mask) - %vec5 = insertelement <8 x i32> %vec4, i32 %res5, i32 5 -; CHECK: vpcmpnlew %zmm1, %zmm0, %k0 {%k1} ## + %ret5 = add i32 %ret4, %res5 %res6 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 6, i32 %mask) - %vec6 = insertelement <8 x i32> %vec5, i32 %res6, i32 6 -; CHECK: vpcmpordw %zmm1, %zmm0, %k0 {%k1} ## + %ret6 = add i32 %ret5, %res6 %res7 = call i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 7, i32 %mask) - %vec7 = insertelement <8 x i32> %vec6, i32 %res7, i32 7 - ret <8 x i32> %vec7 + %ret7 = add i32 %ret6, %res7 + ret i32 %ret7 } declare i32 @llvm.x86.avx512.mask.cmp.w.512(<32 x i16>, <32 x i16>, i32, i32) nounwind readnone -define <8 x i32> @test_ucmp_w_512(<32 x i16> %a0, <32 x i16> %a1) { -; CHECK_LABEL: test_ucmp_w_512 -; CHECK: vpcmpequw %zmm1, %zmm0, %k0 ## +define i32 @test_ucmp_w_512(<32 x i16> %a0, <32 x i16> %a1) { +; AVX512BW-LABEL: test_ucmp_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpcmpequw %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: vpcmpltuw %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %ecx +; AVX512BW-NEXT: addl %eax, %ecx +; AVX512BW-NEXT: vpcmpleuw %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: addl %ecx, %eax +; AVX512BW-NEXT: vpcmpunorduw %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %ecx +; AVX512BW-NEXT: addl %eax, %ecx +; AVX512BW-NEXT: vpcmpnequw %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: addl %ecx, %eax +; AVX512BW-NEXT: vpcmpnltuw %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %ecx +; AVX512BW-NEXT: addl %eax, %ecx +; AVX512BW-NEXT: vpcmpnleuw %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %edx +; AVX512BW-NEXT: addl %ecx, %edx +; AVX512BW-NEXT: vpcmporduw %zmm1, %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: addl %edx, %eax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_ucmp_w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpcmpequw %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: vpcmpltuw %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %ecx +; AVX512F-32-NEXT: addl %eax, %ecx +; AVX512F-32-NEXT: vpcmpleuw %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: addl %ecx, %eax +; AVX512F-32-NEXT: vpcmpunorduw %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %ecx +; AVX512F-32-NEXT: addl %eax, %ecx +; AVX512F-32-NEXT: vpcmpnequw %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: addl %ecx, %eax +; AVX512F-32-NEXT: vpcmpnltuw %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %ecx +; AVX512F-32-NEXT: addl %eax, %ecx +; AVX512F-32-NEXT: vpcmpnleuw %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %edx +; AVX512F-32-NEXT: addl %ecx, %edx +; AVX512F-32-NEXT: vpcmporduw %zmm1, %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: addl %edx, %eax +; AVX512F-32-NEXT: retl %res0 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 0, i32 -1) - %vec0 = insertelement <8 x i32> undef, i32 %res0, i32 0 -; CHECK: vpcmpltuw %zmm1, %zmm0, %k0 ## %res1 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 1, i32 -1) - %vec1 = insertelement <8 x i32> %vec0, i32 %res1, i32 1 -; CHECK: vpcmpleuw %zmm1, %zmm0, %k0 ## + %ret1 = add i32 %res0, %res1 %res2 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 2, i32 -1) - %vec2 = insertelement <8 x i32> %vec1, i32 %res2, i32 2 -; CHECK: vpcmpunorduw %zmm1, %zmm0, %k0 ## + %ret2 = add i32 %ret1, %res2 %res3 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 3, i32 -1) - %vec3 = insertelement <8 x i32> %vec2, i32 %res3, i32 3 -; CHECK: vpcmpnequw %zmm1, %zmm0, %k0 ## + %ret3 = add i32 %ret2, %res3 %res4 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 4, i32 -1) - %vec4 = insertelement <8 x i32> %vec3, i32 %res4, i32 4 -; CHECK: vpcmpnltuw %zmm1, %zmm0, %k0 ## + %ret4 = add i32 %ret3, %res4 %res5 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 5, i32 -1) - %vec5 = insertelement <8 x i32> %vec4, i32 %res5, i32 5 -; CHECK: vpcmpnleuw %zmm1, %zmm0, %k0 ## + %ret5 = add i32 %ret4, %res5 %res6 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 6, i32 -1) - %vec6 = insertelement <8 x i32> %vec5, i32 %res6, i32 6 -; CHECK: vpcmporduw %zmm1, %zmm0, %k0 ## + %ret6 = add i32 %ret5, %res6 %res7 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 7, i32 -1) - %vec7 = insertelement <8 x i32> %vec6, i32 %res7, i32 7 - ret <8 x i32> %vec7 -} - -define <8 x i32> @test_mask_ucmp_w_512(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) { -; CHECK_LABEL: test_mask_ucmp_w_512 -; CHECK: vpcmpequw %zmm1, %zmm0, %k0 {%k1} ## + %ret7 = add i32 %ret6, %res7 + ret i32 %ret7 +} + +define i32 @test_mask_ucmp_w_512(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) { +; AVX512BW-LABEL: test_mask_ucmp_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpcmpequw %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: vpcmpltuw %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %ecx +; AVX512BW-NEXT: addl %eax, %ecx +; AVX512BW-NEXT: vpcmpleuw %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: addl %ecx, %eax +; AVX512BW-NEXT: vpcmpunorduw %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %ecx +; AVX512BW-NEXT: addl %eax, %ecx +; AVX512BW-NEXT: vpcmpnequw %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: addl %ecx, %eax +; AVX512BW-NEXT: vpcmpnltuw %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %ecx +; AVX512BW-NEXT: addl %eax, %ecx +; AVX512BW-NEXT: vpcmpnleuw %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %edx +; AVX512BW-NEXT: addl %ecx, %edx +; AVX512BW-NEXT: vpcmporduw %zmm1, %zmm0, %k0 {%k1} +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: addl %edx, %eax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_ucmp_w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpcmpequw %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: vpcmpltuw %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %ecx +; AVX512F-32-NEXT: addl %eax, %ecx +; AVX512F-32-NEXT: vpcmpleuw %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: addl %ecx, %eax +; AVX512F-32-NEXT: vpcmpunorduw %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %ecx +; AVX512F-32-NEXT: addl %eax, %ecx +; AVX512F-32-NEXT: vpcmpnequw %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: addl %ecx, %eax +; AVX512F-32-NEXT: vpcmpnltuw %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %ecx +; AVX512F-32-NEXT: addl %eax, %ecx +; AVX512F-32-NEXT: vpcmpnleuw %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %edx +; AVX512F-32-NEXT: addl %ecx, %edx +; AVX512F-32-NEXT: vpcmporduw %zmm1, %zmm0, %k0 {%k1} +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: addl %edx, %eax +; AVX512F-32-NEXT: retl %res0 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 0, i32 %mask) - %vec0 = insertelement <8 x i32> undef, i32 %res0, i32 0 -; CHECK: vpcmpltuw %zmm1, %zmm0, %k0 {%k1} ## %res1 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 1, i32 %mask) - %vec1 = insertelement <8 x i32> %vec0, i32 %res1, i32 1 -; CHECK: vpcmpleuw %zmm1, %zmm0, %k0 {%k1} ## + %ret1 = add i32 %res0, %res1 %res2 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 2, i32 %mask) - %vec2 = insertelement <8 x i32> %vec1, i32 %res2, i32 2 -; CHECK: vpcmpunorduw %zmm1, %zmm0, %k0 {%k1} ## + %ret2 = add i32 %ret1, %res2 %res3 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 3, i32 %mask) - %vec3 = insertelement <8 x i32> %vec2, i32 %res3, i32 3 -; CHECK: vpcmpnequw %zmm1, %zmm0, %k0 {%k1} ## + %ret3 = add i32 %ret2, %res3 %res4 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 4, i32 %mask) - %vec4 = insertelement <8 x i32> %vec3, i32 %res4, i32 4 -; CHECK: vpcmpnltuw %zmm1, %zmm0, %k0 {%k1} ## + %ret4 = add i32 %ret3, %res4 %res5 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 5, i32 %mask) - %vec5 = insertelement <8 x i32> %vec4, i32 %res5, i32 5 -; CHECK: vpcmpnleuw %zmm1, %zmm0, %k0 {%k1} ## + %ret5 = add i32 %ret4, %res5 %res6 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 6, i32 %mask) - %vec6 = insertelement <8 x i32> %vec5, i32 %res6, i32 6 -; CHECK: vpcmporduw %zmm1, %zmm0, %k0 {%k1} ## + %ret6 = add i32 %ret5, %res6 %res7 = call i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16> %a0, <32 x i16> %a1, i32 7, i32 %mask) - %vec7 = insertelement <8 x i32> %vec6, i32 %res7, i32 7 - ret <8 x i32> %vec7 + %ret7 = add i32 %ret6, %res7 + ret i32 %ret7 } declare i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16>, <32 x i16>, i32, i32) nounwind readnone -; CHECK-LABEL: test_x86_mask_blend_b_256 -; CHECK: vpblendmb -define <32 x i8> @test_x86_mask_blend_b_256(i32 %a0, <32 x i8> %a1, <32 x i8> %a2) { - %res = call <32 x i8> @llvm.x86.avx512.mask.blend.b.256(<32 x i8> %a1, <32 x i8> %a2, i32 %a0) ; <<32 x i8>> [#uses=1] - ret <32 x i8> %res -} -declare <32 x i8> @llvm.x86.avx512.mask.blend.b.256(<32 x i8>, <32 x i8>, i32) nounwind readonly - -; CHECK-LABEL: test_x86_mask_blend_w_256 -define <16 x i16> @test_x86_mask_blend_w_256(i16 %mask, <16 x i16> %a1, <16 x i16> %a2) { - ; CHECK: vpblendmw - %res = call <16 x i16> @llvm.x86.avx512.mask.blend.w.256(<16 x i16> %a1, <16 x i16> %a2, i16 %mask) ; <<16 x i16>> [#uses=1] - ret <16 x i16> %res -} -declare <16 x i16> @llvm.x86.avx512.mask.blend.w.256(<16 x i16>, <16 x i16>, i16) nounwind readonly - -; CHECK-LABEL: test_x86_mask_blend_b_512 -; CHECK: vpblendmb -define <64 x i8> @test_x86_mask_blend_b_512(i64 %a0, <64 x i8> %a1, <64 x i8> %a2) { - %res = call <64 x i8> @llvm.x86.avx512.mask.blend.b.512(<64 x i8> %a1, <64 x i8> %a2, i64 %a0) ; <<64 x i8>> [#uses=1] - ret <64 x i8> %res -} declare <64 x i8> @llvm.x86.avx512.mask.blend.b.512(<64 x i8>, <64 x i8>, i64) nounwind readonly -; CHECK-LABEL: test_x86_mask_blend_w_512 define <32 x i16> @test_x86_mask_blend_w_512(i32 %mask, <32 x i16> %a1, <32 x i16> %a2) { - ; CHECK: vpblendmw - %res = call <32 x i16> @llvm.x86.avx512.mask.blend.w.512(<32 x i16> %a1, <32 x i16> %a2, i32 %mask) ; <<32 x i16>> [#uses=1] +; AVX512BW-LABEL: test_x86_mask_blend_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpblendmw %zmm1, %zmm0, %zmm0 {%k1} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_x86_mask_blend_w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpblendmw %zmm1, %zmm0, %zmm0 {%k1} +; AVX512F-32-NEXT: retl + %res = call <32 x i16> @llvm.x86.avx512.mask.blend.w.512(<32 x i16> %a1, <32 x i16> %a2, i32 %mask) ; <<32 x i16>> [#uses=1] ret <32 x i16> %res } declare <32 x i16> @llvm.x86.avx512.mask.blend.w.512(<32 x i16>, <32 x i16>, i32) nounwind readonly -; CHECK-LABEL: test_x86_mask_blend_b_128 -; CHECK: vpblendmb -define <16 x i8> @test_x86_mask_blend_b_128(i16 %a0, <16 x i8> %a1, <16 x i8> %a2) { - %res = call <16 x i8> @llvm.x86.avx512.mask.blend.b.128(<16 x i8> %a1, <16 x i8> %a2, i16 %a0) ; <<16 x i8>> [#uses=1] - ret <16 x i8> %res -} -declare <16 x i8> @llvm.x86.avx512.mask.blend.b.128(<16 x i8>, <16 x i8>, i16) nounwind readonly - -; CHECK-LABEL: test_x86_mask_blend_w_128 -define <8 x i16> @test_x86_mask_blend_w_128(i8 %mask, <8 x i16> %a1, <8 x i16> %a2) { - ; CHECK: vpblendmw - %res = call <8 x i16> @llvm.x86.avx512.mask.blend.w.128(<8 x i16> %a1, <8 x i16> %a2, i8 %mask) ; <<8 x i16>> [#uses=1] - ret <8 x i16> %res +define <64 x i8> @test_x86_mask_blend_b_512(i64 %a0, <64 x i8> %a1, <64 x i8> %a2) { +; AVX512BW-LABEL: test_x86_mask_blend_b_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpblendmb %zmm1, %zmm0, %zmm0 {%k1} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_x86_mask_blend_b_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpblendmb %zmm1, %zmm0, %zmm0 {%k1} +; AVX512F-32-NEXT: retl + %res = call <64 x i8> @llvm.x86.avx512.mask.blend.b.512(<64 x i8> %a1, <64 x i8> %a2, i64 %a0) ; <<64 x i8>> [#uses=1] + ret <64 x i8> %res } -declare <8 x i16> @llvm.x86.avx512.mask.blend.w.128(<8 x i16>, <8 x i16>, i8) nounwind readonly define <32 x i16> @test_mask_packs_epi32_rr_512(<16 x i32> %a, <16 x i32> %b) { - ;CHECK-LABEL: test_mask_packs_epi32_rr_512 - ;CHECK: vpackssdw %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0x6b,0xc1] +; AVX512BW-LABEL: test_mask_packs_epi32_rr_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpackssdw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packs_epi32_rr_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpackssdw %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 -1) ret <32 x i16> %res } define <32 x i16> @test_mask_packs_epi32_rrk_512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask) { - ;CHECK-LABEL: test_mask_packs_epi32_rrk_512 - ;CHECK: vpackssdw %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0x6b,0xd1] +; AVX512BW-LABEL: test_mask_packs_epi32_rrk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpackssdw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vmovaps %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packs_epi32_rrk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpackssdw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_packs_epi32_rrkz_512(<16 x i32> %a, <16 x i32> %b, i32 %mask) { - ;CHECK-LABEL: test_mask_packs_epi32_rrkz_512 - ;CHECK: vpackssdw %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x6b,0xc1] +; AVX512BW-LABEL: test_mask_packs_epi32_rrkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpackssdw %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packs_epi32_rrkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpackssdw %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_packs_epi32_rm_512(<16 x i32> %a, <16 x i32>* %ptr_b) { - ;CHECK-LABEL: test_mask_packs_epi32_rm_512 - ;CHECK: vpackssdw (%rdi), %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0x6b,0x07] +; AVX512BW-LABEL: test_mask_packs_epi32_rm_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpackssdw (%rdi), %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packs_epi32_rm_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: vpackssdw (%eax), %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 -1) ret <32 x i16> %res } define <32 x i16> @test_mask_packs_epi32_rmk_512(<16 x i32> %a, <16 x i32>* %ptr_b, <32 x i16> %passThru, i32 %mask) { - ;CHECK-LABEL: test_mask_packs_epi32_rmk_512 - ;CHECK: vpackssdw (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0x6b,0x0f] +; AVX512BW-LABEL: test_mask_packs_epi32_rmk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpackssdw (%rdi), %zmm0, %zmm1 {%k1} +; AVX512BW-NEXT: vmovaps %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packs_epi32_rmk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpackssdw (%eax), %zmm0, %zmm1 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_packs_epi32_rmkz_512(<16 x i32> %a, <16 x i32>* %ptr_b, i32 %mask) { - ;CHECK-LABEL: test_mask_packs_epi32_rmkz_512 - ;CHECK: vpackssdw (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x6b,0x07] +; AVX512BW-LABEL: test_mask_packs_epi32_rmkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpackssdw (%rdi), %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packs_epi32_rmkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpackssdw (%eax), %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_packs_epi32_rmb_512(<16 x i32> %a, i32* %ptr_b) { - ;CHECK-LABEL: test_mask_packs_epi32_rmb_512 - ;CHECK: vpackssdw (%rdi){1to16}, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x58,0x6b,0x07] +; AVX512BW-LABEL: test_mask_packs_epi32_rmb_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpackssdw (%rdi){1to16}, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packs_epi32_rmb_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: vpackssdw (%eax){1to16}, %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %q = load i32, i32* %ptr_b %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0 %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer @@ -408,8 +980,20 @@ define <32 x i16> @test_mask_packs_epi32_rmb_512(<16 x i32> %a, i32* %ptr_b) { } define <32 x i16> @test_mask_packs_epi32_rmbk_512(<16 x i32> %a, i32* %ptr_b, <32 x i16> %passThru, i32 %mask) { - ;CHECK-LABEL: test_mask_packs_epi32_rmbk_512 - ;CHECK: vpackssdw (%rdi){1to16}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x59,0x6b,0x0f] +; AVX512BW-LABEL: test_mask_packs_epi32_rmbk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpackssdw (%rdi){1to16}, %zmm0, %zmm1 {%k1} +; AVX512BW-NEXT: vmovaps %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packs_epi32_rmbk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpackssdw (%eax){1to16}, %zmm0, %zmm1 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %q = load i32, i32* %ptr_b %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0 %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer @@ -418,8 +1002,18 @@ define <32 x i16> @test_mask_packs_epi32_rmbk_512(<16 x i32> %a, i32* %ptr_b, <3 } define <32 x i16> @test_mask_packs_epi32_rmbkz_512(<16 x i32> %a, i32* %ptr_b, i32 %mask) { - ;CHECK-LABEL: test_mask_packs_epi32_rmbkz_512 - ;CHECK: vpackssdw (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xd9,0x6b,0x07] +; AVX512BW-LABEL: test_mask_packs_epi32_rmbkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpackssdw (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packs_epi32_rmbkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpackssdw (%eax){1to16}, %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %q = load i32, i32* %ptr_b %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0 %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer @@ -430,45 +1024,110 @@ define <32 x i16> @test_mask_packs_epi32_rmbkz_512(<16 x i32> %a, i32* %ptr_b, i declare <32 x i16> @llvm.x86.avx512.mask.packssdw.512(<16 x i32>, <16 x i32>, <32 x i16>, i32) define <64 x i8> @test_mask_packs_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) { - ;CHECK-LABEL: test_mask_packs_epi16_rr_512 - ;CHECK: vpacksswb %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x63,0xc1] +; AVX512BW-LABEL: test_mask_packs_epi16_rr_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpacksswb %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packs_epi16_rr_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpacksswb %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 -1) ret <64 x i8> %res } define <64 x i8> @test_mask_packs_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask) { - ;CHECK-LABEL: test_mask_packs_epi16_rrk_512 - ;CHECK: vpacksswb %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0x63,0xd1] +; AVX512BW-LABEL: test_mask_packs_epi16_rrk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpacksswb %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vmovaps %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packs_epi16_rrk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpacksswb %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask) ret <64 x i8> %res } define <64 x i8> @test_mask_packs_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i64 %mask) { - ;CHECK-LABEL: test_mask_packs_epi16_rrkz_512 - ;CHECK: vpacksswb %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x63,0xc1] +; AVX512BW-LABEL: test_mask_packs_epi16_rrkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpacksswb %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packs_epi16_rrkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpacksswb %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 %mask) ret <64 x i8> %res } define <64 x i8> @test_mask_packs_epi16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) { - ;CHECK-LABEL: test_mask_packs_epi16_rm_512 - ;CHECK: vpacksswb (%rdi), %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x63,0x07] +; AVX512BW-LABEL: test_mask_packs_epi16_rm_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpacksswb (%rdi), %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packs_epi16_rm_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: vpacksswb (%eax), %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 -1) ret <64 x i8> %res } define <64 x i8> @test_mask_packs_epi16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <64 x i8> %passThru, i64 %mask) { - ;CHECK-LABEL: test_mask_packs_epi16_rmk_512 - ;CHECK: vpacksswb (%rdi), %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0x63,0x0f] +; AVX512BW-LABEL: test_mask_packs_epi16_rmk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rsi, %k1 +; AVX512BW-NEXT: vpacksswb (%rdi), %zmm0, %zmm1 {%k1} +; AVX512BW-NEXT: vmovaps %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packs_epi16_rmk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpacksswb (%eax), %zmm0, %zmm1 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask) ret <64 x i8> %res } define <64 x i8> @test_mask_packs_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i64 %mask) { - ;CHECK-LABEL: test_mask_packs_epi16_rmkz_512 - ;CHECK: vpacksswb (%rdi), %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x63,0x07] +; AVX512BW-LABEL: test_mask_packs_epi16_rmkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rsi, %k1 +; AVX512BW-NEXT: vpacksswb (%rdi), %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packs_epi16_rmkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpacksswb (%eax), %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 %mask) ret <64 x i8> %res @@ -478,53 +1137,118 @@ declare <64 x i8> @llvm.x86.avx512.mask.packsswb.512(<32 x i16>, <32 x i16>, <64 define <32 x i16> @test_mask_packus_epi32_rr_512(<16 x i32> %a, <16 x i32> %b) { - ;CHECK-LABEL: test_mask_packus_epi32_rr_512 - ;CHECK: vpackusdw %zmm1, %zmm0, %zmm0 +; AVX512BW-LABEL: test_mask_packus_epi32_rr_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpackusdw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packus_epi32_rr_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpackusdw %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 -1) ret <32 x i16> %res } define <32 x i16> @test_mask_packus_epi32_rrk_512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask) { - ;CHECK-LABEL: test_mask_packus_epi32_rrk_512 - ;CHECK: vpackusdw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-LABEL: test_mask_packus_epi32_rrk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpackusdw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vmovaps %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packus_epi32_rrk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpackusdw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_packus_epi32_rrkz_512(<16 x i32> %a, <16 x i32> %b, i32 %mask) { - ;CHECK-LABEL: test_mask_packus_epi32_rrkz_512 - ;CHECK: vpackusdw %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512BW-LABEL: test_mask_packus_epi32_rrkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpackusdw %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packus_epi32_rrkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpackusdw %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_packus_epi32_rm_512(<16 x i32> %a, <16 x i32>* %ptr_b) { - ;CHECK-LABEL: test_mask_packus_epi32_rm_512 - ;CHECK: vpackusdw (%rdi), %zmm0, %zmm0 +; AVX512BW-LABEL: test_mask_packus_epi32_rm_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpackusdw (%rdi), %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packus_epi32_rm_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: vpackusdw (%eax), %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 -1) ret <32 x i16> %res } define <32 x i16> @test_mask_packus_epi32_rmk_512(<16 x i32> %a, <16 x i32>* %ptr_b, <32 x i16> %passThru, i32 %mask) { - ;CHECK-LABEL: test_mask_packus_epi32_rmk_512 - ;CHECK: vpackusdw (%rdi), %zmm0, %zmm1 {%k1} +; AVX512BW-LABEL: test_mask_packus_epi32_rmk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpackusdw (%rdi), %zmm0, %zmm1 {%k1} +; AVX512BW-NEXT: vmovaps %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packus_epi32_rmk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpackusdw (%eax), %zmm0, %zmm1 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> %passThru, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_packus_epi32_rmkz_512(<16 x i32> %a, <16 x i32>* %ptr_b, i32 %mask) { - ;CHECK-LABEL: test_mask_packus_epi32_rmkz_512 - ;CHECK: vpackusdw (%rdi), %zmm0, %zmm0 {%k1} {z} +; AVX512BW-LABEL: test_mask_packus_epi32_rmkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpackusdw (%rdi), %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packus_epi32_rmkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpackusdw (%eax), %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %b = load <16 x i32>, <16 x i32>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32> %a, <16 x i32> %b, <32 x i16> zeroinitializer, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_packus_epi32_rmb_512(<16 x i32> %a, i32* %ptr_b) { - ;CHECK-LABEL: test_mask_packus_epi32_rmb_512 - ;CHECK: vpackusdw (%rdi){1to16}, %zmm0, %zmm0 +; AVX512BW-LABEL: test_mask_packus_epi32_rmb_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpackusdw (%rdi){1to16}, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packus_epi32_rmb_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: vpackusdw (%eax){1to16}, %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %q = load i32, i32* %ptr_b %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0 %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer @@ -533,8 +1257,20 @@ define <32 x i16> @test_mask_packus_epi32_rmb_512(<16 x i32> %a, i32* %ptr_b) { } define <32 x i16> @test_mask_packus_epi32_rmbk_512(<16 x i32> %a, i32* %ptr_b, <32 x i16> %passThru, i32 %mask) { - ;CHECK-LABEL: test_mask_packus_epi32_rmbk_512 - ;CHECK: vpackusdw (%rdi){1to16}, %zmm0, %zmm1 {%k1} +; AVX512BW-LABEL: test_mask_packus_epi32_rmbk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpackusdw (%rdi){1to16}, %zmm0, %zmm1 {%k1} +; AVX512BW-NEXT: vmovaps %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packus_epi32_rmbk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpackusdw (%eax){1to16}, %zmm0, %zmm1 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %q = load i32, i32* %ptr_b %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0 %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer @@ -543,8 +1279,18 @@ define <32 x i16> @test_mask_packus_epi32_rmbk_512(<16 x i32> %a, i32* %ptr_b, < } define <32 x i16> @test_mask_packus_epi32_rmbkz_512(<16 x i32> %a, i32* %ptr_b, i32 %mask) { - ;CHECK-LABEL: test_mask_packus_epi32_rmbkz_512 - ;CHECK: vpackusdw (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} +; AVX512BW-LABEL: test_mask_packus_epi32_rmbkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpackusdw (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packus_epi32_rmbkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpackusdw (%eax){1to16}, %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %q = load i32, i32* %ptr_b %vecinit.i = insertelement <16 x i32> undef, i32 %q, i32 0 %b = shufflevector <16 x i32> %vecinit.i, <16 x i32> undef, <16 x i32> zeroinitializer @@ -555,45 +1301,110 @@ define <32 x i16> @test_mask_packus_epi32_rmbkz_512(<16 x i32> %a, i32* %ptr_b, declare <32 x i16> @llvm.x86.avx512.mask.packusdw.512(<16 x i32>, <16 x i32>, <32 x i16>, i32) define <64 x i8> @test_mask_packus_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) { - ;CHECK-LABEL: test_mask_packus_epi16_rr_512 - ;CHECK: vpackuswb %zmm1, %zmm0, %zmm0 +; AVX512BW-LABEL: test_mask_packus_epi16_rr_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpackuswb %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packus_epi16_rr_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpackuswb %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 -1) ret <64 x i8> %res } define <64 x i8> @test_mask_packus_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask) { - ;CHECK-LABEL: test_mask_packus_epi16_rrk_512 - ;CHECK: vpackuswb %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-LABEL: test_mask_packus_epi16_rrk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpackuswb %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vmovaps %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packus_epi16_rrk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpackuswb %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask) ret <64 x i8> %res } define <64 x i8> @test_mask_packus_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i64 %mask) { - ;CHECK-LABEL: test_mask_packus_epi16_rrkz_512 - ;CHECK: vpackuswb %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512BW-LABEL: test_mask_packus_epi16_rrkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpackuswb %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packus_epi16_rrkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpackuswb %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 %mask) ret <64 x i8> %res } define <64 x i8> @test_mask_packus_epi16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) { - ;CHECK-LABEL: test_mask_packus_epi16_rm_512 - ;CHECK: vpackuswb (%rdi), %zmm0, %zmm0 +; AVX512BW-LABEL: test_mask_packus_epi16_rm_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpackuswb (%rdi), %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packus_epi16_rm_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: vpackuswb (%eax), %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 -1) ret <64 x i8> %res } define <64 x i8> @test_mask_packus_epi16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <64 x i8> %passThru, i64 %mask) { - ;CHECK-LABEL: test_mask_packus_epi16_rmk_512 - ;CHECK: vpackuswb (%rdi), %zmm0, %zmm1 {%k1} +; AVX512BW-LABEL: test_mask_packus_epi16_rmk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rsi, %k1 +; AVX512BW-NEXT: vpackuswb (%rdi), %zmm0, %zmm1 {%k1} +; AVX512BW-NEXT: vmovaps %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packus_epi16_rmk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpackuswb (%eax), %zmm0, %zmm1 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> %passThru, i64 %mask) ret <64 x i8> %res } define <64 x i8> @test_mask_packus_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i64 %mask) { - ;CHECK-LABEL: test_mask_packus_epi16_rmkz_512 - ;CHECK: vpackuswb (%rdi), %zmm0, %zmm0 {%k1} {z} +; AVX512BW-LABEL: test_mask_packus_epi16_rmkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rsi, %k1 +; AVX512BW-NEXT: vpackuswb (%rdi), %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_packus_epi16_rmkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpackuswb (%eax), %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16> %a, <32 x i16> %b, <64 x i8> zeroinitializer, i64 %mask) ret <64 x i8> %res @@ -602,45 +1413,102 @@ define <64 x i8> @test_mask_packus_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %pt declare <64 x i8> @llvm.x86.avx512.mask.packuswb.512(<32 x i16>, <32 x i16>, <64 x i8>, i64) define <32 x i16> @test_mask_adds_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) { - ;CHECK-LABEL: test_mask_adds_epi16_rr_512 - ;CHECK: vpaddsw %zmm1, %zmm0, %zmm0 +; AVX512BW-LABEL: test_mask_adds_epi16_rr_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpaddsw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_adds_epi16_rr_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpaddsw %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1) ret <32 x i16> %res } define <32 x i16> @test_mask_adds_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) { - ;CHECK-LABEL: test_mask_adds_epi16_rrk_512 - ;CHECK: vpaddsw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-LABEL: test_mask_adds_epi16_rrk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpaddsw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vmovaps %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_adds_epi16_rrk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpaddsw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_adds_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) { - ;CHECK-LABEL: test_mask_adds_epi16_rrkz_512 - ;CHECK: vpaddsw %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512BW-LABEL: test_mask_adds_epi16_rrkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpaddsw %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_adds_epi16_rrkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpaddsw %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_adds_epi16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) { - ;CHECK-LABEL: test_mask_adds_epi16_rm_512 - ;CHECK: vpaddsw (%rdi), %zmm0, %zmm0 +; AVX512BW-LABEL: test_mask_adds_epi16_rm_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpaddsw (%rdi), %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_adds_epi16_rm_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: vpaddsw (%eax), %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1) ret <32 x i16> %res } define <32 x i16> @test_mask_adds_epi16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) { - ;CHECK-LABEL: test_mask_adds_epi16_rmk_512 - ;CHECK: vpaddsw (%rdi), %zmm0, %zmm1 {%k1} +; AVX512BW-LABEL: test_mask_adds_epi16_rmk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpaddsw (%rdi), %zmm0, %zmm1 {%k1} +; AVX512BW-NEXT: vmovaps %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_adds_epi16_rmk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpaddsw (%eax), %zmm0, %zmm1 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_adds_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) { - ;CHECK-LABEL: test_mask_adds_epi16_rmkz_512 - ;CHECK: vpaddsw (%rdi), %zmm0, %zmm0 {%k1} {z} +; AVX512BW-LABEL: test_mask_adds_epi16_rmkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpaddsw (%rdi), %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_adds_epi16_rmkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpaddsw (%eax), %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask) ret <32 x i16> %res @@ -649,45 +1517,102 @@ define <32 x i16> @test_mask_adds_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr declare <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) define <32 x i16> @test_mask_subs_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) { - ;CHECK-LABEL: test_mask_subs_epi16_rr_512 - ;CHECK: vpsubsw %zmm1, %zmm0, %zmm0 +; AVX512BW-LABEL: test_mask_subs_epi16_rr_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsubsw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_subs_epi16_rr_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpsubsw %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1) ret <32 x i16> %res } define <32 x i16> @test_mask_subs_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) { - ;CHECK-LABEL: test_mask_subs_epi16_rrk_512 - ;CHECK: vpsubsw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-LABEL: test_mask_subs_epi16_rrk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpsubsw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vmovaps %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_subs_epi16_rrk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpsubsw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_subs_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) { - ;CHECK-LABEL: test_mask_subs_epi16_rrkz_512 - ;CHECK: vpsubsw %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512BW-LABEL: test_mask_subs_epi16_rrkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpsubsw %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_subs_epi16_rrkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpsubsw %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_subs_epi16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) { - ;CHECK-LABEL: test_mask_subs_epi16_rm_512 - ;CHECK: vpsubsw (%rdi), %zmm0, %zmm0 +; AVX512BW-LABEL: test_mask_subs_epi16_rm_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsubsw (%rdi), %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_subs_epi16_rm_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: vpsubsw (%eax), %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1) ret <32 x i16> %res } define <32 x i16> @test_mask_subs_epi16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) { - ;CHECK-LABEL: test_mask_subs_epi16_rmk_512 - ;CHECK: vpsubsw (%rdi), %zmm0, %zmm1 {%k1} +; AVX512BW-LABEL: test_mask_subs_epi16_rmk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpsubsw (%rdi), %zmm0, %zmm1 {%k1} +; AVX512BW-NEXT: vmovaps %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_subs_epi16_rmk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpsubsw (%eax), %zmm0, %zmm1 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_subs_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) { - ;CHECK-LABEL: test_mask_subs_epi16_rmkz_512 - ;CHECK: vpsubsw (%rdi), %zmm0, %zmm0 {%k1} {z} +; AVX512BW-LABEL: test_mask_subs_epi16_rmkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpsubsw (%rdi), %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_subs_epi16_rmkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpsubsw (%eax), %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask) ret <32 x i16> %res @@ -696,45 +1621,102 @@ define <32 x i16> @test_mask_subs_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr declare <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) define <32 x i16> @test_mask_adds_epu16_rr_512(<32 x i16> %a, <32 x i16> %b) { - ;CHECK-LABEL: test_mask_adds_epu16_rr_512 - ;CHECK: vpaddusw %zmm1, %zmm0, %zmm0 +; AVX512BW-LABEL: test_mask_adds_epu16_rr_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpaddusw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_adds_epu16_rr_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpaddusw %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1) ret <32 x i16> %res } define <32 x i16> @test_mask_adds_epu16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) { - ;CHECK-LABEL: test_mask_adds_epu16_rrk_512 - ;CHECK: vpaddusw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-LABEL: test_mask_adds_epu16_rrk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpaddusw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vmovaps %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_adds_epu16_rrk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpaddusw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_adds_epu16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) { - ;CHECK-LABEL: test_mask_adds_epu16_rrkz_512 - ;CHECK: vpaddusw %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512BW-LABEL: test_mask_adds_epu16_rrkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpaddusw %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_adds_epu16_rrkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpaddusw %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_adds_epu16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) { - ;CHECK-LABEL: test_mask_adds_epu16_rm_512 - ;CHECK: vpaddusw (%rdi), %zmm0, %zmm0 +; AVX512BW-LABEL: test_mask_adds_epu16_rm_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpaddusw (%rdi), %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_adds_epu16_rm_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: vpaddusw (%eax), %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1) ret <32 x i16> %res } define <32 x i16> @test_mask_adds_epu16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) { - ;CHECK-LABEL: test_mask_adds_epu16_rmk_512 - ;CHECK: vpaddusw (%rdi), %zmm0, %zmm1 {%k1} +; AVX512BW-LABEL: test_mask_adds_epu16_rmk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpaddusw (%rdi), %zmm0, %zmm1 {%k1} +; AVX512BW-NEXT: vmovaps %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_adds_epu16_rmk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpaddusw (%eax), %zmm0, %zmm1 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_adds_epu16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) { - ;CHECK-LABEL: test_mask_adds_epu16_rmkz_512 - ;CHECK: vpaddusw (%rdi), %zmm0, %zmm0 {%k1} {z} +; AVX512BW-LABEL: test_mask_adds_epu16_rmkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpaddusw (%rdi), %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_adds_epu16_rmkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpaddusw (%eax), %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask) ret <32 x i16> %res @@ -743,45 +1725,102 @@ define <32 x i16> @test_mask_adds_epu16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr declare <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) define <32 x i16> @test_mask_subs_epu16_rr_512(<32 x i16> %a, <32 x i16> %b) { - ;CHECK-LABEL: test_mask_subs_epu16_rr_512 - ;CHECK: vpsubusw %zmm1, %zmm0, %zmm0 +; AVX512BW-LABEL: test_mask_subs_epu16_rr_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsubusw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_subs_epu16_rr_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpsubusw %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1) ret <32 x i16> %res } define <32 x i16> @test_mask_subs_epu16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) { - ;CHECK-LABEL: test_mask_subs_epu16_rrk_512 - ;CHECK: vpsubusw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-LABEL: test_mask_subs_epu16_rrk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpsubusw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vmovaps %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_subs_epu16_rrk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpsubusw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_subs_epu16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) { - ;CHECK-LABEL: test_mask_subs_epu16_rrkz_512 - ;CHECK: vpsubusw %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512BW-LABEL: test_mask_subs_epu16_rrkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpsubusw %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_subs_epu16_rrkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpsubusw %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_subs_epu16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) { - ;CHECK-LABEL: test_mask_subs_epu16_rm_512 - ;CHECK: vpsubusw (%rdi), %zmm0, %zmm0 +; AVX512BW-LABEL: test_mask_subs_epu16_rm_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsubusw (%rdi), %zmm0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_subs_epu16_rm_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: vpsubusw (%eax), %zmm0, %zmm0 +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 -1) ret <32 x i16> %res } define <32 x i16> @test_mask_subs_epu16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) { - ;CHECK-LABEL: test_mask_subs_epu16_rmk_512 - ;CHECK: vpsubusw (%rdi), %zmm0, %zmm1 {%k1} +; AVX512BW-LABEL: test_mask_subs_epu16_rmk_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpsubusw (%rdi), %zmm0, %zmm1 {%k1} +; AVX512BW-NEXT: vmovaps %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_subs_epu16_rmk_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpsubusw (%eax), %zmm0, %zmm1 {%k1} +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) ret <32 x i16> %res } define <32 x i16> @test_mask_subs_epu16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) { - ;CHECK-LABEL: test_mask_subs_epu16_rmkz_512 - ;CHECK: vpsubusw (%rdi), %zmm0, %zmm0 {%k1} {z} +; AVX512BW-LABEL: test_mask_subs_epu16_rmkz_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpsubusw (%rdi), %zmm0, %zmm0 {%k1} {z} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_mask_subs_epu16_rmkz_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpsubusw (%eax), %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: retl %b = load <32 x i16>, <32 x i16>* %ptr_b %res = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a, <32 x i16> %b, <32 x i16> zeroinitializer, i32 %mask) ret <32 x i16> %res @@ -791,11 +1830,24 @@ declare <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16>, <32 x i16>, <3 declare <64 x i8> @llvm.x86.avx512.mask.pmaxs.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64) -; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_b_512 -; CHECK-NOT: call -; CHECK: vpmaxsb %zmm -; CHECK: {%k1} define <64 x i8>@test_int_x86_avx512_mask_pmaxs_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pmaxs_b_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpmaxsb %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpmaxsb %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddb %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pmaxs_b_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpmaxsb %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vpmaxsb %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddb %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <64 x i8> @llvm.x86.avx512.mask.pmaxs.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) %res1 = call <64 x i8> @llvm.x86.avx512.mask.pmaxs.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1) %res2 = add <64 x i8> %res, %res1 @@ -804,11 +1856,22 @@ define <64 x i8>@test_int_x86_avx512_mask_pmaxs_b_512(<64 x i8> %x0, <64 x i8> % declare <32 x i16> @llvm.x86.avx512.mask.pmaxs.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) -; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_w_512 -; CHECK-NOT: call -; CHECK: vpmaxsw %zmm -; CHECK: {%k1} define <32 x i16>@test_int_x86_avx512_mask_pmaxs_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pmaxs_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpmaxsw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpmaxsw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pmaxs_w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpmaxsw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vpmaxsw %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.pmaxs.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) %res1 = call <32 x i16> @llvm.x86.avx512.mask.pmaxs.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1) %res2 = add <32 x i16> %res, %res1 @@ -817,11 +1880,24 @@ define <32 x i16>@test_int_x86_avx512_mask_pmaxs_w_512(<32 x i16> %x0, <32 x i16 declare <64 x i8> @llvm.x86.avx512.mask.pmaxu.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64) -; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_b_512 -; CHECK-NOT: call -; CHECK: vpmaxub %zmm -; CHECK: {%k1} define <64 x i8>@test_int_x86_avx512_mask_pmaxu_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pmaxu_b_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpmaxub %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpmaxub %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddb %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pmaxu_b_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpmaxub %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vpmaxub %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddb %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <64 x i8> @llvm.x86.avx512.mask.pmaxu.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) %res1 = call <64 x i8> @llvm.x86.avx512.mask.pmaxu.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1) %res2 = add <64 x i8> %res, %res1 @@ -830,11 +1906,22 @@ define <64 x i8>@test_int_x86_avx512_mask_pmaxu_b_512(<64 x i8> %x0, <64 x i8> % declare <32 x i16> @llvm.x86.avx512.mask.pmaxu.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) -; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_w_512 -; CHECK-NOT: call -; CHECK: vpmaxuw %zmm -; CHECK: {%k1} define <32 x i16>@test_int_x86_avx512_mask_pmaxu_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pmaxu_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpmaxuw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpmaxuw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pmaxu_w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpmaxuw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vpmaxuw %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.pmaxu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) %res1 = call <32 x i16> @llvm.x86.avx512.mask.pmaxu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1) %res2 = add <32 x i16> %res, %res1 @@ -843,11 +1930,24 @@ define <32 x i16>@test_int_x86_avx512_mask_pmaxu_w_512(<32 x i16> %x0, <32 x i16 declare <64 x i8> @llvm.x86.avx512.mask.pmins.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64) -; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_b_512 -; CHECK-NOT: call -; CHECK: vpminsb %zmm -; CHECK: {%k1} define <64 x i8>@test_int_x86_avx512_mask_pmins_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pmins_b_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpminsb %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpminsb %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddb %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pmins_b_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpminsb %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vpminsb %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddb %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <64 x i8> @llvm.x86.avx512.mask.pmins.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) %res1 = call <64 x i8> @llvm.x86.avx512.mask.pmins.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1) %res2 = add <64 x i8> %res, %res1 @@ -856,11 +1956,22 @@ define <64 x i8>@test_int_x86_avx512_mask_pmins_b_512(<64 x i8> %x0, <64 x i8> % declare <32 x i16> @llvm.x86.avx512.mask.pmins.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) -; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_w_512 -; CHECK-NOT: call -; CHECK: vpminsw %zmm -; CHECK: {%k1} define <32 x i16>@test_int_x86_avx512_mask_pmins_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pmins_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpminsw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpminsw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pmins_w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpminsw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vpminsw %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.pmins.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) %res1 = call <32 x i16> @llvm.x86.avx512.mask.pmins.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1) %res2 = add <32 x i16> %res, %res1 @@ -869,11 +1980,24 @@ define <32 x i16>@test_int_x86_avx512_mask_pmins_w_512(<32 x i16> %x0, <32 x i16 declare <64 x i8> @llvm.x86.avx512.mask.pminu.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64) -; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_b_512 -; CHECK-NOT: call -; CHECK: vpminub %zmm -; CHECK: {%k1} define <64 x i8>@test_int_x86_avx512_mask_pminu_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pminu_b_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpminub %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpminub %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddb %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pminu_b_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpminub %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vpminub %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddb %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <64 x i8> @llvm.x86.avx512.mask.pminu.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) %res1 = call <64 x i8> @llvm.x86.avx512.mask.pminu.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1) %res2 = add <64 x i8> %res, %res1 @@ -882,11 +2006,22 @@ define <64 x i8>@test_int_x86_avx512_mask_pminu_b_512(<64 x i8> %x0, <64 x i8> % declare <32 x i16> @llvm.x86.avx512.mask.pminu.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) -; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_w_512 -; CHECK-NOT: call -; CHECK: vpminuw %zmm -; CHECK: {%k1} define <32 x i16>@test_int_x86_avx512_mask_pminu_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pminu_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpminuw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpminuw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pminu_w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpminuw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vpminuw %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.pminu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) %res1 = call <32 x i16> @llvm.x86.avx512.mask.pminu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1) %res2 = add <32 x i16> %res, %res1 @@ -895,11 +2030,24 @@ define <32 x i16>@test_int_x86_avx512_mask_pminu_w_512(<32 x i16> %x0, <32 x i16 declare <32 x i16> @llvm.x86.avx512.mask.vpermt2var.hi.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) -; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_hi_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpermt2w %zmm{{.*}}{%k1} define <32 x i16>@test_int_x86_avx512_mask_vpermt2var_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_vpermt2var_hi_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vmovaps %zmm1, %zmm3 +; AVX512BW-NEXT: vpermt2w %zmm2, %zmm0, %zmm3 {%k1} +; AVX512BW-NEXT: vpermt2w %zmm2, %zmm0, %zmm1 +; AVX512BW-NEXT: vpaddw %zmm1, %zmm3, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_vpermt2var_hi_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm3 +; AVX512F-32-NEXT: vpermt2w %zmm2, %zmm0, %zmm3 {%k1} +; AVX512F-32-NEXT: vpermt2w %zmm2, %zmm0, %zmm1 +; AVX512F-32-NEXT: vpaddw %zmm1, %zmm3, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.vpermt2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpermt2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1) %res2 = add <32 x i16> %res, %res1 @@ -908,11 +2056,24 @@ define <32 x i16>@test_int_x86_avx512_mask_vpermt2var_hi_512(<32 x i16> %x0, <32 declare <32 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) -; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_hi_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpermt2w %zmm{{.*}}{%k1} {z} define <32 x i16>@test_int_x86_avx512_maskz_vpermt2var_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_maskz_vpermt2var_hi_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vmovaps %zmm1, %zmm3 +; AVX512BW-NEXT: vpermt2w %zmm2, %zmm0, %zmm3 {%k1} {z} +; AVX512BW-NEXT: vpermt2w %zmm2, %zmm0, %zmm1 +; AVX512BW-NEXT: vpaddw %zmm1, %zmm3, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_maskz_vpermt2var_hi_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm3 +; AVX512F-32-NEXT: vpermt2w %zmm2, %zmm0, %zmm3 {%k1} {z} +; AVX512F-32-NEXT: vpermt2w %zmm2, %zmm0, %zmm1 +; AVX512F-32-NEXT: vpaddw %zmm1, %zmm3, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) %res1 = call <32 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1) %res2 = add <32 x i16> %res, %res1 @@ -921,11 +2082,24 @@ define <32 x i16>@test_int_x86_avx512_maskz_vpermt2var_hi_512(<32 x i16> %x0, <3 declare <32 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) -; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_hi_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpermi2w %zmm{{.*}}{%k1} define <32 x i16>@test_int_x86_avx512_mask_vpermi2var_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_vpermi2var_hi_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vmovaps %zmm1, %zmm3 +; AVX512BW-NEXT: vpermi2w %zmm2, %zmm0, %zmm3 {%k1} +; AVX512BW-NEXT: vpermi2w %zmm2, %zmm0, %zmm1 +; AVX512BW-NEXT: vpaddw %zmm1, %zmm3, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_vpermi2var_hi_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm3 +; AVX512F-32-NEXT: vpermi2w %zmm2, %zmm0, %zmm3 {%k1} +; AVX512F-32-NEXT: vpermi2w %zmm2, %zmm0, %zmm1 +; AVX512F-32-NEXT: vpaddw %zmm1, %zmm3, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1) %res2 = add <32 x i16> %res, %res1 @@ -934,11 +2108,24 @@ define <32 x i16>@test_int_x86_avx512_mask_vpermi2var_hi_512(<32 x i16> %x0, <32 declare <64 x i8> @llvm.x86.avx512.mask.pavg.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64) -; CHECK-LABEL: @test_int_x86_avx512_mask_pavg_b_512 -; CHECK-NOT: call -; CHECK: vpavgb %zmm -; CHECK: {%k1} define <64 x i8>@test_int_x86_avx512_mask_pavg_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pavg_b_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpavgb %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpavgb %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddb %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pavg_b_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpavgb %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vpavgb %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddb %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <64 x i8> @llvm.x86.avx512.mask.pavg.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) %res1 = call <64 x i8> @llvm.x86.avx512.mask.pavg.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1) %res2 = add <64 x i8> %res, %res1 @@ -947,11 +2134,22 @@ define <64 x i8>@test_int_x86_avx512_mask_pavg_b_512(<64 x i8> %x0, <64 x i8> %x declare <32 x i16> @llvm.x86.avx512.mask.pavg.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) -; CHECK-LABEL: @test_int_x86_avx512_mask_pavg_w_512 -; CHECK-NOT: call -; CHECK: vpavgw %zmm -; CHECK: {%k1} define <32 x i16>@test_int_x86_avx512_mask_pavg_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pavg_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpavgw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpavgw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pavg_w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpavgw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vpavgw %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.pavg.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) %res1 = call <32 x i16> @llvm.x86.avx512.mask.pavg.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1) %res2 = add <32 x i16> %res, %res1 @@ -960,11 +2158,24 @@ define <32 x i16>@test_int_x86_avx512_mask_pavg_w_512(<32 x i16> %x0, <32 x i16> declare <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64) -; CHECK-LABEL: @test_int_x86_avx512_mask_pshuf_b_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpshufb %zmm{{.*}}{%k1} define <64 x i8>@test_int_x86_avx512_mask_pshuf_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pshuf_b_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpshufb %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpshufb %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddb %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pshuf_b_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpshufb %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vpshufb %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddb %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) %res1 = call <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1) %res2 = add <64 x i8> %res, %res1 @@ -973,11 +2184,22 @@ define <64 x i8>@test_int_x86_avx512_mask_pshuf_b_512(<64 x i8> %x0, <64 x i8> % declare <32 x i16> @llvm.x86.avx512.mask.pabs.w.512(<32 x i16>, <32 x i16>, i32) -; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_w_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpabsw{{.*}}{%k1} define <32 x i16>@test_int_x86_avx512_mask_pabs_w_512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pabs_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpabsw %zmm0, %zmm1 {%k1} +; AVX512BW-NEXT: vpabsw %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddw %zmm0, %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pabs_w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpabsw %zmm0, %zmm1 {%k1} +; AVX512F-32-NEXT: vpabsw %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddw %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.pabs.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2) %res1 = call <32 x i16> @llvm.x86.avx512.mask.pabs.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 -1) %res2 = add <32 x i16> %res, %res1 @@ -986,11 +2208,24 @@ define <32 x i16>@test_int_x86_avx512_mask_pabs_w_512(<32 x i16> %x0, <32 x i16> declare <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8>, <64 x i8>, i64) -; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_b_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpabsb{{.*}}{%k1} define <64 x i8>@test_int_x86_avx512_mask_pabs_b_512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pabs_b_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpabsb %zmm0, %zmm1 {%k1} +; AVX512BW-NEXT: vpabsb %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddb %zmm0, %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pabs_b_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpabsb %zmm0, %zmm1 {%k1} +; AVX512F-32-NEXT: vpabsb %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddb %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %res = call <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2) %res1 = call <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 -1) %res2 = add <64 x i8> %res, %res1 @@ -999,12 +2234,22 @@ define <64 x i8>@test_int_x86_avx512_mask_pabs_b_512(<64 x i8> %x0, <64 x i8> %x declare <32 x i16> @llvm.x86.avx512.mask.pmulhu.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) -; CHECK-LABEL: @test_int_x86_avx512_mask_pmulhu_w_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: {%k1} -; CHECK: vpmulhuw {{.*}}encoding: [0x62 define <32 x i16>@test_int_x86_avx512_mask_pmulhu_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pmulhu_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpmulhuw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpmulhuw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pmulhu_w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpmulhuw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vpmulhuw %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.pmulhu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) %res1 = call <32 x i16> @llvm.x86.avx512.mask.pmulhu.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1) %res2 = add <32 x i16> %res, %res1 @@ -1013,12 +2258,22 @@ define <32 x i16>@test_int_x86_avx512_mask_pmulhu_w_512(<32 x i16> %x0, <32 x i1 declare <32 x i16> @llvm.x86.avx512.mask.pmulh.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) -; CHECK-LABEL: @test_int_x86_avx512_mask_pmulh_w_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: {%k1} -; CHECK: vpmulhw {{.*}}encoding: [0x62 define <32 x i16>@test_int_x86_avx512_mask_pmulh_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pmulh_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpmulhw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpmulhw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pmulh_w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpmulhw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vpmulhw %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.pmulh.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) %res1 = call <32 x i16> @llvm.x86.avx512.mask.pmulh.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1) %res2 = add <32 x i16> %res, %res1 @@ -1027,14 +2282,627 @@ define <32 x i16>@test_int_x86_avx512_mask_pmulh_w_512(<32 x i16> %x0, <32 x i16 declare <32 x i16> @llvm.x86.avx512.mask.pmul.hr.sw.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) -; CHECK-LABEL: @test_int_x86_avx512_mask_pmulhr_sw_512 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: {%k1} -; CHECK: vpmulhrsw {{.*}}encoding: [0x62 define <32 x i16>@test_int_x86_avx512_mask_pmulhr_sw_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pmulhr_sw_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpmulhrsw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpmulhrsw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pmulhr_sw_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpmulhrsw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vpmulhrsw %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %res = call <32 x i16> @llvm.x86.avx512.mask.pmul.hr.sw.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) %res1 = call <32 x i16> @llvm.x86.avx512.mask.pmul.hr.sw.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1) %res2 = add <32 x i16> %res, %res1 ret <32 x i16> %res2 } + +declare <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16>, <32 x i8>, i32) + +define <32 x i8>@test_int_x86_avx512_mask_pmov_wb_512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pmov_wb_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpmovwb %zmm0, %ymm1 {%k1} +; AVX512BW-NEXT: vpmovwb %zmm0, %ymm2 {%k1} {z} +; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0 +; AVX512BW-NEXT: vpaddb %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: vpaddb %ymm2, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pmov_wb_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpmovwb %zmm0, %ymm1 {%k1} +; AVX512F-32-NEXT: vpmovwb %zmm0, %ymm2 {%k1} {z} +; AVX512F-32-NEXT: vpmovwb %zmm0, %ymm0 +; AVX512F-32-NEXT: vpaddb %ymm1, %ymm0, %ymm0 +; AVX512F-32-NEXT: vpaddb %ymm2, %ymm0, %ymm0 +; AVX512F-32-NEXT: retl + %res0 = call <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 -1) + %res1 = call <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2) + %res2 = call <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16> %x0, <32 x i8> zeroinitializer, i32 %x2) + %res3 = add <32 x i8> %res0, %res1 + %res4 = add <32 x i8> %res3, %res2 + ret <32 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.wb.mem.512(i8* %ptr, <32 x i16>, i32) + +define void @test_int_x86_avx512_mask_pmov_wb_mem_512(i8* %ptr, <32 x i16> %x1, i32 %x2) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pmov_wb_mem_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpmovwb %zmm0, (%rdi) +; AVX512BW-NEXT: vpmovwb %zmm0, (%rdi) {%k1} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pmov_wb_mem_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: vpmovwb %zmm0, (%eax) +; AVX512F-32-NEXT: vpmovwb %zmm0, (%eax) {%k1} +; AVX512F-32-NEXT: retl + call void @llvm.x86.avx512.mask.pmov.wb.mem.512(i8* %ptr, <32 x i16> %x1, i32 -1) + call void @llvm.x86.avx512.mask.pmov.wb.mem.512(i8* %ptr, <32 x i16> %x1, i32 %x2) + ret void +} + +declare <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16>, <32 x i8>, i32) + +define <32 x i8>@test_int_x86_avx512_mask_pmovs_wb_512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pmovs_wb_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpmovswb %zmm0, %ymm1 {%k1} +; AVX512BW-NEXT: vpmovswb %zmm0, %ymm2 {%k1} {z} +; AVX512BW-NEXT: vpmovswb %zmm0, %ymm0 +; AVX512BW-NEXT: vpaddb %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: vpaddb %ymm2, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pmovs_wb_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpmovswb %zmm0, %ymm1 {%k1} +; AVX512F-32-NEXT: vpmovswb %zmm0, %ymm2 {%k1} {z} +; AVX512F-32-NEXT: vpmovswb %zmm0, %ymm0 +; AVX512F-32-NEXT: vpaddb %ymm1, %ymm0, %ymm0 +; AVX512F-32-NEXT: vpaddb %ymm2, %ymm0, %ymm0 +; AVX512F-32-NEXT: retl + %res0 = call <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 -1) + %res1 = call <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2) + %res2 = call <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16> %x0, <32 x i8> zeroinitializer, i32 %x2) + %res3 = add <32 x i8> %res0, %res1 + %res4 = add <32 x i8> %res3, %res2 + ret <32 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.wb.mem.512(i8* %ptr, <32 x i16>, i32) + +define void @test_int_x86_avx512_mask_pmovs_wb_mem_512(i8* %ptr, <32 x i16> %x1, i32 %x2) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pmovs_wb_mem_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpmovswb %zmm0, (%rdi) +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpmovswb %zmm0, (%rdi) {%k1} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pmovs_wb_mem_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %ecx +; AVX512F-32-NEXT: vpmovswb %zmm0, (%ecx) +; AVX512F-32-NEXT: kmovd %eax, %k1 +; AVX512F-32-NEXT: vpmovswb %zmm0, (%ecx) {%k1} +; AVX512F-32-NEXT: retl + call void @llvm.x86.avx512.mask.pmovs.wb.mem.512(i8* %ptr, <32 x i16> %x1, i32 -1) + call void @llvm.x86.avx512.mask.pmovs.wb.mem.512(i8* %ptr, <32 x i16> %x1, i32 %x2) + ret void +} + +declare <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16>, <32 x i8>, i32) + +define <32 x i8>@test_int_x86_avx512_mask_pmovus_wb_512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pmovus_wb_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpmovuswb %zmm0, %ymm1 {%k1} +; AVX512BW-NEXT: vpmovuswb %zmm0, %ymm2 {%k1} {z} +; AVX512BW-NEXT: vpmovuswb %zmm0, %ymm0 +; AVX512BW-NEXT: vpaddb %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: vpaddb %ymm2, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pmovus_wb_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpmovuswb %zmm0, %ymm1 {%k1} +; AVX512F-32-NEXT: vpmovuswb %zmm0, %ymm2 {%k1} {z} +; AVX512F-32-NEXT: vpmovuswb %zmm0, %ymm0 +; AVX512F-32-NEXT: vpaddb %ymm1, %ymm0, %ymm0 +; AVX512F-32-NEXT: vpaddb %ymm2, %ymm0, %ymm0 +; AVX512F-32-NEXT: retl + %res0 = call <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 -1) + %res1 = call <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2) + %res2 = call <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16> %x0, <32 x i8> zeroinitializer, i32 %x2) + %res3 = add <32 x i8> %res0, %res1 + %res4 = add <32 x i8> %res3, %res2 + ret <32 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.wb.mem.512(i8* %ptr, <32 x i16>, i32) + +define void @test_int_x86_avx512_mask_pmovus_wb_mem_512(i8* %ptr, <32 x i16> %x1, i32 %x2) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pmovus_wb_mem_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpmovuswb %zmm0, (%rdi) +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpmovuswb %zmm0, (%rdi) {%k1} +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pmovus_wb_mem_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %ecx +; AVX512F-32-NEXT: vpmovuswb %zmm0, (%ecx) +; AVX512F-32-NEXT: kmovd %eax, %k1 +; AVX512F-32-NEXT: vpmovuswb %zmm0, (%ecx) {%k1} +; AVX512F-32-NEXT: retl + call void @llvm.x86.avx512.mask.pmovus.wb.mem.512(i8* %ptr, <32 x i16> %x1, i32 -1) + call void @llvm.x86.avx512.mask.pmovus.wb.mem.512(i8* %ptr, <32 x i16> %x1, i32 %x2) + ret void +} + +declare <32 x i16> @llvm.x86.avx512.mask.pmaddubs.w.512(<64 x i8>, <64 x i8>, <32 x i16>, i32) + +define <32 x i16>@test_int_x86_avx512_mask_pmaddubs_w_512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x2, i32 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pmaddubs_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pmaddubs_w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl + %res = call <32 x i16> @llvm.x86.avx512.mask.pmaddubs.w.512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x2, i32 %x3) + %res1 = call <32 x i16> @llvm.x86.avx512.mask.pmaddubs.w.512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x2, i32 -1) + %res2 = add <32 x i16> %res, %res1 + ret <32 x i16> %res2 +} + +declare <16 x i32> @llvm.x86.avx512.mask.pmaddw.d.512(<32 x i16>, <32 x i16>, <16 x i32>, i16) + +define <16 x i32>@test_int_x86_avx512_mask_pmaddw_d_512(<32 x i16> %x0, <32 x i16> %x1, <16 x i32> %x2, i16 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_pmaddw_d_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovw %edi, %k1 +; AVX512BW-NEXT: vpmaddwd %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpmaddwd %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_pmaddw_d_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovw {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpmaddwd %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vpmaddwd %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddd %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl + %res = call <16 x i32> @llvm.x86.avx512.mask.pmaddw.d.512(<32 x i16> %x0, <32 x i16> %x1, <16 x i32> %x2, i16 %x3) + %res1 = call <16 x i32> @llvm.x86.avx512.mask.pmaddw.d.512(<32 x i16> %x0, <32 x i16> %x1, <16 x i32> %x2, i16 -1) + %res2 = add <16 x i32> %res, %res1 + ret <16 x i32> %res2 +} + +declare <64 x i8> @llvm.x86.avx512.mask.punpckhb.w.512(<64 x i8>, <64 x i8>, <64 x i8>, i64) + +define <64 x i8>@test_int_x86_avx512_mask_punpckhb_w_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_punpckhb_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm2[8],k1[8],zmm2[9],k1[9],zmm2[10],k1[10],zmm2[11],k1[11],zmm2[12],k1[12],zmm2[13],k1[13],zmm2[14],k1[14],zmm2[15],k1[15],zmm2[24],k1[24],zmm2[25],k1[25],zmm2[26],k1[26],zmm2[27],k1[27],zmm2[28],k1[28],zmm2[29],k1[29],zmm2[30],k1[30],zmm2[31],k1[31],zmm2[40],k1[40],zmm2[41],k1[41],zmm2[42],k1[42],zmm2[43],k1[43],zmm2[44],k1[44],zmm2[45],k1[45],zmm2[46],k1[46],zmm2[47],k1[47],zmm2[56],k1[56],zmm2[57],k1[57],zmm2[58],k1[58],zmm2[59],k1[59],zmm2[60],k1[60],zmm2[61],k1[61],zmm2[62],k1[62],zmm2[63],k1[63] +; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm0 = zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[12],zmm1[12],zmm0[13],zmm1[13],zmm0[14],zmm1[14],zmm0[15],zmm1[15],zmm0[24],zmm1[24],zmm0[25],zmm1[25],zmm0[26],zmm1[26],zmm0[27],zmm1[27],zmm0[28],zmm1[28],zmm0[29],zmm1[29],zmm0[30],zmm1[30],zmm0[31],zmm1[31],zmm0[40],zmm1[40],zmm0[41],zmm1[41],zmm0[42],zmm1[42],zmm0[43],zmm1[43],zmm0[44],zmm1[44],zmm0[45],zmm1[45],zmm0[46],zmm1[46],zmm0[47],zmm1[47],zmm0[56],zmm1[56],zmm0[57],zmm1[57],zmm0[58],zmm1[58],zmm0[59],zmm1[59],zmm0[60],zmm1[60],zmm0[61],zmm1[61],zmm0[62],zmm1[62],zmm0[63],zmm1[63] +; AVX512BW-NEXT: vpaddb %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_punpckhb_w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm2[8],k1[8],zmm2[9],k1[9],zmm2[10],k1[10],zmm2[11],k1[11],zmm2[12],k1[12],zmm2[13],k1[13],zmm2[14],k1[14],zmm2[15],k1[15],zmm2[24],k1[24],zmm2[25],k1[25],zmm2[26],k1[26],zmm2[27],k1[27],zmm2[28],k1[28],zmm2[29],k1[29],zmm2[30],k1[30],zmm2[31],k1[31],zmm2[40],k1[40],zmm2[41],k1[41],zmm2[42],k1[42],zmm2[43],k1[43],zmm2[44],k1[44],zmm2[45],k1[45],zmm2[46],k1[46],zmm2[47],k1[47],zmm2[56],k1[56],zmm2[57],k1[57],zmm2[58],k1[58],zmm2[59],k1[59],zmm2[60],k1[60],zmm2[61],k1[61],zmm2[62],k1[62],zmm2[63],k1[63] +; AVX512F-32-NEXT: vpunpckhbw {{.*#+}} zmm0 = zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[12],zmm1[12],zmm0[13],zmm1[13],zmm0[14],zmm1[14],zmm0[15],zmm1[15],zmm0[24],zmm1[24],zmm0[25],zmm1[25],zmm0[26],zmm1[26],zmm0[27],zmm1[27],zmm0[28],zmm1[28],zmm0[29],zmm1[29],zmm0[30],zmm1[30],zmm0[31],zmm1[31],zmm0[40],zmm1[40],zmm0[41],zmm1[41],zmm0[42],zmm1[42],zmm0[43],zmm1[43],zmm0[44],zmm1[44],zmm0[45],zmm1[45],zmm0[46],zmm1[46],zmm0[47],zmm1[47],zmm0[56],zmm1[56],zmm0[57],zmm1[57],zmm0[58],zmm1[58],zmm0[59],zmm1[59],zmm0[60],zmm1[60],zmm0[61],zmm1[61],zmm0[62],zmm1[62],zmm0[63],zmm1[63] +; AVX512F-32-NEXT: vpaddb %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl + %res = call <64 x i8> @llvm.x86.avx512.mask.punpckhb.w.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) + %res1 = call <64 x i8> @llvm.x86.avx512.mask.punpckhb.w.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1) + %res2 = add <64 x i8> %res, %res1 + ret <64 x i8> %res2 +} + +declare <64 x i8> @llvm.x86.avx512.mask.punpcklb.w.512(<64 x i8>, <64 x i8>, <64 x i8>, i64) + +define <64 x i8>@test_int_x86_avx512_mask_punpcklb_w_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_punpcklb_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm2 = zmm2[0],k1[0],zmm2[1],k1[1],zmm2[2],k1[2],zmm2[3],k1[3],zmm2[4],k1[4],zmm2[5],k1[5],zmm2[6],k1[6],zmm2[7],k1[7],zmm2[16],k1[16],zmm2[17],k1[17],zmm2[18],k1[18],zmm2[19],k1[19],zmm2[20],k1[20],zmm2[21],k1[21],zmm2[22],k1[22],zmm2[23],k1[23],zmm2[32],k1[32],zmm2[33],k1[33],zmm2[34],k1[34],zmm2[35],k1[35],zmm2[36],k1[36],zmm2[37],k1[37],zmm2[38],k1[38],zmm2[39],k1[39],zmm2[48],k1[48],zmm2[49],k1[49],zmm2[50],k1[50],zmm2[51],k1[51],zmm2[52],k1[52],zmm2[53],k1[53],zmm2[54],k1[54],zmm2[55],k1[55] +; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[16],zmm1[16],zmm0[17],zmm1[17],zmm0[18],zmm1[18],zmm0[19],zmm1[19],zmm0[20],zmm1[20],zmm0[21],zmm1[21],zmm0[22],zmm1[22],zmm0[23],zmm1[23],zmm0[32],zmm1[32],zmm0[33],zmm1[33],zmm0[34],zmm1[34],zmm0[35],zmm1[35],zmm0[36],zmm1[36],zmm0[37],zmm1[37],zmm0[38],zmm1[38],zmm0[39],zmm1[39],zmm0[48],zmm1[48],zmm0[49],zmm1[49],zmm0[50],zmm1[50],zmm0[51],zmm1[51],zmm0[52],zmm1[52],zmm0[53],zmm1[53],zmm0[54],zmm1[54],zmm0[55],zmm1[55] +; AVX512BW-NEXT: vpaddb %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_punpcklb_w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpunpcklbw {{.*#+}} zmm2 = zmm2[0],k1[0],zmm2[1],k1[1],zmm2[2],k1[2],zmm2[3],k1[3],zmm2[4],k1[4],zmm2[5],k1[5],zmm2[6],k1[6],zmm2[7],k1[7],zmm2[16],k1[16],zmm2[17],k1[17],zmm2[18],k1[18],zmm2[19],k1[19],zmm2[20],k1[20],zmm2[21],k1[21],zmm2[22],k1[22],zmm2[23],k1[23],zmm2[32],k1[32],zmm2[33],k1[33],zmm2[34],k1[34],zmm2[35],k1[35],zmm2[36],k1[36],zmm2[37],k1[37],zmm2[38],k1[38],zmm2[39],k1[39],zmm2[48],k1[48],zmm2[49],k1[49],zmm2[50],k1[50],zmm2[51],k1[51],zmm2[52],k1[52],zmm2[53],k1[53],zmm2[54],k1[54],zmm2[55],k1[55] +; AVX512F-32-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[16],zmm1[16],zmm0[17],zmm1[17],zmm0[18],zmm1[18],zmm0[19],zmm1[19],zmm0[20],zmm1[20],zmm0[21],zmm1[21],zmm0[22],zmm1[22],zmm0[23],zmm1[23],zmm0[32],zmm1[32],zmm0[33],zmm1[33],zmm0[34],zmm1[34],zmm0[35],zmm1[35],zmm0[36],zmm1[36],zmm0[37],zmm1[37],zmm0[38],zmm1[38],zmm0[39],zmm1[39],zmm0[48],zmm1[48],zmm0[49],zmm1[49],zmm0[50],zmm1[50],zmm0[51],zmm1[51],zmm0[52],zmm1[52],zmm0[53],zmm1[53],zmm0[54],zmm1[54],zmm0[55],zmm1[55] +; AVX512F-32-NEXT: vpaddb %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl + %res = call <64 x i8> @llvm.x86.avx512.mask.punpcklb.w.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) + %res1 = call <64 x i8> @llvm.x86.avx512.mask.punpcklb.w.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1) + %res2 = add <64 x i8> %res, %res1 + ret <64 x i8> %res2 +} + +declare <32 x i16> @llvm.x86.avx512.mask.punpckhw.d.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) + +define <32 x i16>@test_int_x86_avx512_mask_punpckhw_d_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_punpckhw_d_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpunpckhwd {{.*#+}} zmm2 = zmm2[4],k1[4],zmm2[5],k1[5],zmm2[6],k1[6],zmm2[7],k1[7],zmm2[12],k1[12],zmm2[13],k1[13],zmm2[14],k1[14],zmm2[15],k1[15],zmm2[20],k1[20],zmm2[21],k1[21],zmm2[22],k1[22],zmm2[23],k1[23],zmm2[28],k1[28],zmm2[29],k1[29],zmm2[30],k1[30],zmm2[31],k1[31] +; AVX512BW-NEXT: vpunpckhwd {{.*#+}} zmm0 = zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[12],zmm1[12],zmm0[13],zmm1[13],zmm0[14],zmm1[14],zmm0[15],zmm1[15],zmm0[20],zmm1[20],zmm0[21],zmm1[21],zmm0[22],zmm1[22],zmm0[23],zmm1[23],zmm0[28],zmm1[28],zmm0[29],zmm1[29],zmm0[30],zmm1[30],zmm0[31],zmm1[31] +; AVX512BW-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_punpckhw_d_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpunpckhwd {{.*#+}} zmm2 = zmm2[4],k1[4],zmm2[5],k1[5],zmm2[6],k1[6],zmm2[7],k1[7],zmm2[12],k1[12],zmm2[13],k1[13],zmm2[14],k1[14],zmm2[15],k1[15],zmm2[20],k1[20],zmm2[21],k1[21],zmm2[22],k1[22],zmm2[23],k1[23],zmm2[28],k1[28],zmm2[29],k1[29],zmm2[30],k1[30],zmm2[31],k1[31] +; AVX512F-32-NEXT: vpunpckhwd {{.*#+}} zmm0 = zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[12],zmm1[12],zmm0[13],zmm1[13],zmm0[14],zmm1[14],zmm0[15],zmm1[15],zmm0[20],zmm1[20],zmm0[21],zmm1[21],zmm0[22],zmm1[22],zmm0[23],zmm1[23],zmm0[28],zmm1[28],zmm0[29],zmm1[29],zmm0[30],zmm1[30],zmm0[31],zmm1[31] +; AVX512F-32-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl + %res = call <32 x i16> @llvm.x86.avx512.mask.punpckhw.d.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) + %res1 = call <32 x i16> @llvm.x86.avx512.mask.punpckhw.d.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1) + %res2 = add <32 x i16> %res, %res1 + ret <32 x i16> %res2 +} + +declare <32 x i16> @llvm.x86.avx512.mask.punpcklw.d.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) + +define <32 x i16>@test_int_x86_avx512_mask_punpcklw_d_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_punpcklw_d_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpunpcklwd {{.*#+}} zmm2 = zmm2[0],k1[0],zmm2[1],k1[1],zmm2[2],k1[2],zmm2[3],k1[3],zmm2[8],k1[8],zmm2[9],k1[9],zmm2[10],k1[10],zmm2[11],k1[11],zmm2[16],k1[16],zmm2[17],k1[17],zmm2[18],k1[18],zmm2[19],k1[19],zmm2[24],k1[24],zmm2[25],k1[25],zmm2[26],k1[26],zmm2[27],k1[27] +; AVX512BW-NEXT: vpunpcklwd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[16],zmm1[16],zmm0[17],zmm1[17],zmm0[18],zmm1[18],zmm0[19],zmm1[19],zmm0[24],zmm1[24],zmm0[25],zmm1[25],zmm0[26],zmm1[26],zmm0[27],zmm1[27] +; AVX512BW-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_punpcklw_d_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vpunpcklwd {{.*#+}} zmm2 = zmm2[0],k1[0],zmm2[1],k1[1],zmm2[2],k1[2],zmm2[3],k1[3],zmm2[8],k1[8],zmm2[9],k1[9],zmm2[10],k1[10],zmm2[11],k1[11],zmm2[16],k1[16],zmm2[17],k1[17],zmm2[18],k1[18],zmm2[19],k1[19],zmm2[24],k1[24],zmm2[25],k1[25],zmm2[26],k1[26],zmm2[27],k1[27] +; AVX512F-32-NEXT: vpunpcklwd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[16],zmm1[16],zmm0[17],zmm1[17],zmm0[18],zmm1[18],zmm0[19],zmm1[19],zmm0[24],zmm1[24],zmm0[25],zmm1[25],zmm0[26],zmm1[26],zmm0[27],zmm1[27] +; AVX512F-32-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl + %res = call <32 x i16> @llvm.x86.avx512.mask.punpcklw.d.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) + %res1 = call <32 x i16> @llvm.x86.avx512.mask.punpcklw.d.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1) + %res2 = add <32 x i16> %res, %res1 + ret <32 x i16> %res2 +} + +declare <64 x i8> @llvm.x86.avx512.mask.palignr.512(<64 x i8>, <64 x i8>, i32, <64 x i8>, i64) + +define <64 x i8>@test_int_x86_avx512_mask_palignr_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x3, i64 %x4) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_palignr_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k1 +; AVX512BW-NEXT: vpalignr $2, %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpalignr $2, %zmm1, %zmm0, %zmm3 {%k1} {z} +; AVX512BW-NEXT: vpalignr $2, %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddb %zmm3, %zmm2, %zmm1 +; AVX512BW-NEXT: vpaddb %zmm0, %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_palignr_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpalignr $2, %zmm1, %zmm0, %zmm3 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k1 +; AVX512F-32-NEXT: vpalignr $2, %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vpalignr $2, %zmm1, %zmm0, %zmm0 {%k1} {z} +; AVX512F-32-NEXT: vpaddb %zmm0, %zmm2, %zmm0 +; AVX512F-32-NEXT: vpaddb %zmm3, %zmm0, %zmm0 +; AVX512F-32-NEXT: retl + %res = call <64 x i8> @llvm.x86.avx512.mask.palignr.512(<64 x i8> %x0, <64 x i8> %x1, i32 2, <64 x i8> %x3, i64 %x4) + %res1 = call <64 x i8> @llvm.x86.avx512.mask.palignr.512(<64 x i8> %x0, <64 x i8> %x1, i32 2, <64 x i8> zeroinitializer, i64 %x4) + %res2 = call <64 x i8> @llvm.x86.avx512.mask.palignr.512(<64 x i8> %x0, <64 x i8> %x1, i32 2, <64 x i8> %x3, i64 -1) + %res3 = add <64 x i8> %res, %res1 + %res4 = add <64 x i8> %res3, %res2 + ret <64 x i8> %res4 +} + +declare <32 x i16> @llvm.x86.avx512.mask.dbpsadbw.512(<64 x i8>, <64 x i8>, i32, <32 x i16>, i32) + +define <32 x i16>@test_int_x86_avx512_mask_dbpsadbw_512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x3, i32 %x4) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_dbpsadbw_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vdbpsadbw $2, %zmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vdbpsadbw $2, %zmm1, %zmm0, %zmm3 {%k1} {z} +; AVX512BW-NEXT: vdbpsadbw $2, %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddw %zmm3, %zmm2, %zmm1 +; AVX512BW-NEXT: vpaddw %zmm0, %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_dbpsadbw_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: vdbpsadbw $2, %zmm1, %zmm0, %zmm2 {%k1} +; AVX512F-32-NEXT: vdbpsadbw $2, %zmm1, %zmm0, %zmm3 {%k1} {z} +; AVX512F-32-NEXT: vdbpsadbw $2, %zmm1, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddw %zmm3, %zmm2, %zmm1 +; AVX512F-32-NEXT: vpaddw %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl + %res = call <32 x i16> @llvm.x86.avx512.mask.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 2, <32 x i16> %x3, i32 %x4) + %res1 = call <32 x i16> @llvm.x86.avx512.mask.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 2, <32 x i16> zeroinitializer, i32 %x4) + %res2 = call <32 x i16> @llvm.x86.avx512.mask.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 2, <32 x i16> %x3, i32 -1) + %res3 = add <32 x i16> %res, %res1 + %res4 = add <32 x i16> %res3, %res2 + ret <32 x i16> %res4 +} + +declare <8 x i64> @llvm.x86.avx512.psll.dq.512(<8 x i64>, i32) + +define <8 x i64>@test_int_x86_avx512_mask_psll_dq_512(<8 x i64> %x0) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_psll_dq_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpslldq $8, %zmm0, %zmm1 +; AVX512BW-NEXT: vpslldq $4, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddq %zmm0, %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_psll_dq_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpslldq $8, %zmm0, %zmm1 +; AVX512F-32-NEXT: vpslldq $4, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl + %res = call <8 x i64> @llvm.x86.avx512.psll.dq.512(<8 x i64> %x0, i32 8) + %res1 = call <8 x i64> @llvm.x86.avx512.psll.dq.512(<8 x i64> %x0, i32 4) + %res2 = add <8 x i64> %res, %res1 + ret <8 x i64> %res2 +} + +declare <8 x i64> @llvm.x86.avx512.psrl.dq.512(<8 x i64>, i32) + +define <8 x i64>@test_int_x86_avx512_mask_psrl_dq_512(<8 x i64> %x0) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_psrl_dq_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsrldq $8, %zmm0, %zmm1 +; AVX512BW-NEXT: vpsrldq $4, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddq %zmm0, %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_psrl_dq_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpsrldq $8, %zmm0, %zmm1 +; AVX512F-32-NEXT: vpsrldq $4, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl + %res = call <8 x i64> @llvm.x86.avx512.psrl.dq.512(<8 x i64> %x0, i32 8) + %res1 = call <8 x i64> @llvm.x86.avx512.psrl.dq.512(<8 x i64> %x0, i32 4) + %res2 = add <8 x i64> %res, %res1 + ret <8 x i64> %res2 +} +declare <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8>, <64 x i8>) + +define <8 x i64>@test_int_x86_avx512_mask_psadb_w_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2){ +; AVX512BW-LABEL: test_int_x86_avx512_mask_psadb_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsadbw %zmm1, %zmm0, %zmm1 +; AVX512BW-NEXT: vpsadbw %zmm2, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddq %zmm0, %zmm1, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_mask_psadb_w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpsadbw %zmm1, %zmm0, %zmm1 +; AVX512F-32-NEXT: vpsadbw %zmm2, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpaddq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl + %res = call <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8> %x0, <64 x i8> %x1) + %res1 = call <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8> %x0, <64 x i8> %x2) + %res2 = add <8 x i64> %res, %res1 + ret <8 x i64> %res2 +} + +declare i32 @llvm.x86.avx512.kunpck.wd(i32, i32) + +define i32@test_int_x86_avx512_kunpck_wd(i32 %x0, i32 %x1) { +; AVX512BW-LABEL: test_int_x86_avx512_kunpck_wd: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k0 +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: kunpckwd %k1, %k0, %k0 +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_kunpck_wd: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckwd %k1, %k0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: retl + %res = call i32 @llvm.x86.avx512.kunpck.wd(i32 %x0, i32 %x1) + ret i32 %res +} + +declare i64 @llvm.x86.avx512.kunpck.dq(i64, i64) + +define i64@test_int_x86_avx512_kunpck_qd(i64 %x0, i64 %x1) { +; AVX512BW-LABEL: test_int_x86_avx512_kunpck_qd: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k0 +; AVX512BW-NEXT: kmovq %rsi, %k1 +; AVX512BW-NEXT: kunpckdq %k1, %k0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_kunpck_qd: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: subl $12, %esp +; AVX512F-32-NEXT: .Ltmp8: +; AVX512F-32-NEXT: .cfi_def_cfa_offset 16 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k0 +; AVX512F-32-NEXT: kmovq %k0, (%esp) +; AVX512F-32-NEXT: movl (%esp), %eax +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: addl $12, %esp +; AVX512F-32-NEXT: retl + %res = call i64 @llvm.x86.avx512.kunpck.dq(i64 %x0, i64 %x1) + ret i64 %res +} + +declare i64 @llvm.x86.avx512.cvtb2mask.512(<64 x i8>) + +define i64@test_int_x86_avx512_cvtb2mask_512(<64 x i8> %x0) { +; AVX512BW-LABEL: test_int_x86_avx512_cvtb2mask_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpmovb2m %zmm0, %k0 +; AVX512BW-NEXT: kmovq %k0, %rax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_cvtb2mask_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: subl $12, %esp +; AVX512F-32-NEXT: .Ltmp9: +; AVX512F-32-NEXT: .cfi_def_cfa_offset 16 +; AVX512F-32-NEXT: vpmovb2m %zmm0, %k0 +; AVX512F-32-NEXT: kmovq %k0, (%esp) +; AVX512F-32-NEXT: movl (%esp), %eax +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %edx +; AVX512F-32-NEXT: addl $12, %esp +; AVX512F-32-NEXT: retl + %res = call i64 @llvm.x86.avx512.cvtb2mask.512(<64 x i8> %x0) + ret i64 %res +} + +declare i32 @llvm.x86.avx512.cvtw2mask.512(<32 x i16>) + +define i32@test_int_x86_avx512_cvtw2mask_512(<32 x i16> %x0) { +; AVX512BW-LABEL: test_int_x86_avx512_cvtw2mask_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpmovw2m %zmm0, %k0 +; AVX512BW-NEXT: kmovd %k0, %eax +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_cvtw2mask_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpmovw2m %zmm0, %k0 +; AVX512F-32-NEXT: kmovd %k0, %eax +; AVX512F-32-NEXT: retl + %res = call i32 @llvm.x86.avx512.cvtw2mask.512(<32 x i16> %x0) + ret i32 %res +} + +declare <64 x i8> @llvm.x86.avx512.cvtmask2b.512(i64) + +define <64 x i8>@test_int_x86_avx512_cvtmask2b_512(i64 %x0) { +; AVX512BW-LABEL: test_int_x86_avx512_cvtmask2b_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovq %rdi, %k0 +; AVX512BW-NEXT: vpmovm2b %k0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_cvtmask2b_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k1 +; AVX512F-32-NEXT: kunpckdq %k0, %k1, %k0 +; AVX512F-32-NEXT: vpmovm2b %k0, %zmm0 +; AVX512F-32-NEXT: retl + %res = call <64 x i8> @llvm.x86.avx512.cvtmask2b.512(i64 %x0) + ret <64 x i8> %res +} + +declare <32 x i16> @llvm.x86.avx512.cvtmask2w.512(i32) + +define <32 x i16>@test_int_x86_avx512_cvtmask2w_512(i32 %x0) { +; AVX512BW-LABEL: test_int_x86_avx512_cvtmask2w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k0 +; AVX512BW-NEXT: vpmovm2w %k0, %zmm0 +; AVX512BW-NEXT: retq +; +; AVX512F-32-LABEL: test_int_x86_avx512_cvtmask2w_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: kmovd {{[0-9]+}}(%esp), %k0 +; AVX512F-32-NEXT: vpmovm2w %k0, %zmm0 +; AVX512F-32-NEXT: retl + %res = call <32 x i16> @llvm.x86.avx512.cvtmask2w.512(i32 %x0) + ret <32 x i16> %res +} + +declare <32 x i16> @llvm.x86.avx512.mask.psrl.w.512(<32 x i16>, <8 x i16>, <32 x i16>, i32) + +define <32 x i16>@test_int_x86_avx512_mask_psrl_w_512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_psrl_w_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %edi, %k1 +; AVX512BW-NEXT: vpsrlw %xmm1, %zmm0, %zmm2 {%k1} +; AVX512BW-NEXT: vpsrlw %xmm1, %zmm0, %zmm3 {%k1} {z} +; AVX512BW-NEXT: vpsrlw %xmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddw %zmm0, %zmm2, %zmm0 +; AVX512BW-NEXT: vpaddw %zmm3, %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %res = call <32 x i16> @llvm.x86.avx512.mask.psrl.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 %x3) + %res1 = call <32 x i16> @llvm.x86.avx512.mask.psrl.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 -1) + %res2 = call <32 x i16> @llvm.x86.avx512.mask.psrl.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3) + %res3 = add <32 x i16> %res, %res1 + %res4 = add <32 x i16> %res3, %res2 + ret <32 x i16> %res4 +} + +declare <32 x i16> @llvm.x86.avx512.mask.psrl.wi.512(<32 x i16>, i8, <32 x i16>, i32) + +define <32 x i16>@test_int_x86_avx512_mask_psrl_wi_512(<32 x i16> %x0, i8 %x1, <32 x i16> %x2, i32 %x3) { +; AVX512BW-LABEL: test_int_x86_avx512_mask_psrl_wi_512: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: kmovd %esi, %k1 +; AVX512BW-NEXT: vpsrlw $3, %zmm0, %zmm1 {%k1} +; AVX512BW-NEXT: vpsrlw $3, %zmm0, %zmm2 {%k1} {z} +; AVX512BW-NEXT: vpsrlw $3, %zmm0, %zmm0 +; AVX512BW-NEXT: vpaddw %zmm0, %zmm1, %zmm0 +; AVX512BW-NEXT: vpaddw %zmm2, %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %res = call <32 x i16> @llvm.x86.avx512.mask.psrl.wi.512(<32 x i16> %x0, i8 3, <32 x i16> %x2, i32 %x3) + %res1 = call <32 x i16> @llvm.x86.avx512.mask.psrl.wi.512(<32 x i16> %x0, i8 3, <32 x i16> %x2, i32 -1) + %res2 = call <32 x i16> @llvm.x86.avx512.mask.psrl.wi.512(<32 x i16> %x0, i8 3, <32 x i16> zeroinitializer, i32 %x3) + %res3 = add <32 x i16> %res, %res1 + %res4 = add <32 x i16> %res3, %res2 + ret <32 x i16> %res4 +} diff --git a/test/CodeGen/X86/avx512bwvl-intrinsics.ll b/test/CodeGen/X86/avx512bwvl-intrinsics.ll index f5413896789a..1db6756c23a8 100644 --- a/test/CodeGen/X86/avx512bwvl-intrinsics.ll +++ b/test/CodeGen/X86/avx512bwvl-intrinsics.ll @@ -3763,6 +3763,38 @@ define <16 x i16>@test_int_x86_avx512_mask_pabs_w_256(<16 x i16> %x0, <16 x i16> ret <16 x i16> %res2 } +; CHECK-LABEL: test_x86_mask_blend_b_256 +; CHECK: vpblendmb +define <32 x i8> @test_x86_mask_blend_b_256(i32 %a0, <32 x i8> %a1, <32 x i8> %a2) { + %res = call <32 x i8> @llvm.x86.avx512.mask.blend.b.256(<32 x i8> %a1, <32 x i8> %a2, i32 %a0) ; <<32 x i8>> [#uses=1] + ret <32 x i8> %res +} +declare <32 x i8> @llvm.x86.avx512.mask.blend.b.256(<32 x i8>, <32 x i8>, i32) nounwind readonly + +; CHECK-LABEL: test_x86_mask_blend_w_256 +define <16 x i16> @test_x86_mask_blend_w_256(i16 %mask, <16 x i16> %a1, <16 x i16> %a2) { + ; CHECK: vpblendmw + %res = call <16 x i16> @llvm.x86.avx512.mask.blend.w.256(<16 x i16> %a1, <16 x i16> %a2, i16 %mask) ; <<16 x i16>> [#uses=1] + ret <16 x i16> %res +} +declare <16 x i16> @llvm.x86.avx512.mask.blend.w.256(<16 x i16>, <16 x i16>, i16) nounwind readonly + +; CHECK-LABEL: test_x86_mask_blend_b_128 +; CHECK: vpblendmb +define <16 x i8> @test_x86_mask_blend_b_128(i16 %a0, <16 x i8> %a1, <16 x i8> %a2) { + %res = call <16 x i8> @llvm.x86.avx512.mask.blend.b.128(<16 x i8> %a1, <16 x i8> %a2, i16 %a0) ; <<16 x i8>> [#uses=1] + ret <16 x i8> %res +} +declare <16 x i8> @llvm.x86.avx512.mask.blend.b.128(<16 x i8>, <16 x i8>, i16) nounwind readonly + +; CHECK-LABEL: test_x86_mask_blend_w_128 +define <8 x i16> @test_x86_mask_blend_w_128(i8 %mask, <8 x i16> %a1, <8 x i16> %a2) { + ; CHECK: vpblendmw + %res = call <8 x i16> @llvm.x86.avx512.mask.blend.w.128(<8 x i16> %a1, <8 x i16> %a2, i8 %mask) ; <<8 x i16>> [#uses=1] + ret <8 x i16> %res +} +declare <8 x i16> @llvm.x86.avx512.mask.blend.w.128(<8 x i16>, <8 x i16>, i8) nounwind readonly + declare <8 x i16> @llvm.x86.avx512.mask.pmulhu.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pmulhu_w_128 @@ -3843,3 +3875,719 @@ define <16 x i16>@test_int_x86_avx512_mask_pmulhr_sw_256(<16 x i16> %x0, <16 x i %res2 = add <16 x i16> %res, %res1 ret <16 x i16> %res2 } + +declare <16 x i8> @llvm.x86.avx512.mask.pmov.wb.128(<8 x i16>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmov_wb_128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_wb_128: +; CHECK: vpmovwb %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovwb %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovwb %xmm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.128(<8 x i16> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.wb.mem.128(i8* %ptr, <8 x i16>, i8) + +define void @test_int_x86_avx512_mask_pmov_wb_mem_128(i8* %ptr, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_wb_mem_128: +; CHECK: vpmovwb %xmm0, (%rdi) +; CHECK: vpmovwb %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmov.wb.mem.128(i8* %ptr, <8 x i16> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmov.wb.mem.128(i8* %ptr, <8 x i16> %x1, i8 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.128(<8 x i16>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmovs_wb_128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_wb_128: +; CHECK: vpmovswb %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovswb %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovswb %xmm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.128(<8 x i16> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.wb.mem.128(i8* %ptr, <8 x i16>, i8) + +define void @test_int_x86_avx512_mask_pmovs_wb_mem_128(i8* %ptr, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_wb_mem_128: +; CHECK: vpmovswb %xmm0, (%rdi) +; CHECK: vpmovswb %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovs.wb.mem.128(i8* %ptr, <8 x i16> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovs.wb.mem.128(i8* %ptr, <8 x i16> %x1, i8 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.128(<8 x i16>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmovus_wb_128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_wb_128: +; CHECK: vpmovuswb %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovuswb %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovuswb %xmm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.128(<8 x i16> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.wb.mem.128(i8* %ptr, <8 x i16>, i8) + +define void @test_int_x86_avx512_mask_pmovus_wb_mem_128(i8* %ptr, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_wb_mem_128: +; CHECK: vpmovuswb %xmm0, (%rdi) +; CHECK: vpmovuswb %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovus.wb.mem.128(i8* %ptr, <8 x i16> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovus.wb.mem.128(i8* %ptr, <8 x i16> %x1, i8 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16>, <16 x i8>, i16) + +define <16 x i8>@test_int_x86_avx512_mask_pmov_wb_256(<16 x i16> %x0, <16 x i8> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_wb_256: +; CHECK: vpmovwb %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovwb %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovwb %ymm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16> %x0, <16 x i8> %x1, i16 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16> %x0, <16 x i8> %x1, i16 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16> %x0, <16 x i8> zeroinitializer, i16 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.wb.mem.256(i8* %ptr, <16 x i16>, i16) + +define void @test_int_x86_avx512_mask_pmov_wb_mem_256(i8* %ptr, <16 x i16> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_wb_mem_256: +; CHECK: vpmovwb %ymm0, (%rdi) +; CHECK: vpmovwb %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmov.wb.mem.256(i8* %ptr, <16 x i16> %x1, i16 -1) + call void @llvm.x86.avx512.mask.pmov.wb.mem.256(i8* %ptr, <16 x i16> %x1, i16 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.256(<16 x i16>, <16 x i8>, i16) + +define <16 x i8>@test_int_x86_avx512_mask_pmovs_wb_256(<16 x i16> %x0, <16 x i8> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_wb_256: +; CHECK: vpmovswb %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovswb %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovswb %ymm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.256(<16 x i16> %x0, <16 x i8> %x1, i16 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.256(<16 x i16> %x0, <16 x i8> %x1, i16 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.256(<16 x i16> %x0, <16 x i8> zeroinitializer, i16 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.wb.mem.256(i8* %ptr, <16 x i16>, i16) + +define void @test_int_x86_avx512_mask_pmovs_wb_mem_256(i8* %ptr, <16 x i16> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_wb_mem_256: +; CHECK: vpmovswb %ymm0, (%rdi) +; CHECK: vpmovswb %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovs.wb.mem.256(i8* %ptr, <16 x i16> %x1, i16 -1) + call void @llvm.x86.avx512.mask.pmovs.wb.mem.256(i8* %ptr, <16 x i16> %x1, i16 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.256(<16 x i16>, <16 x i8>, i16) + +define <16 x i8>@test_int_x86_avx512_mask_pmovus_wb_256(<16 x i16> %x0, <16 x i8> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_wb_256: +; CHECK: vpmovuswb %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovuswb %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovuswb %ymm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.256(<16 x i16> %x0, <16 x i8> %x1, i16 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.256(<16 x i16> %x0, <16 x i8> %x1, i16 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.256(<16 x i16> %x0, <16 x i8> zeroinitializer, i16 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.wb.mem.256(i8* %ptr, <16 x i16>, i16) + +define void @test_int_x86_avx512_mask_pmovus_wb_mem_256(i8* %ptr, <16 x i16> %x1, i16 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_wb_mem_256: +; CHECK: vpmovuswb %ymm0, (%rdi) +; CHECK: vpmovuswb %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovus.wb.mem.256(i8* %ptr, <16 x i16> %x1, i16 -1) + call void @llvm.x86.avx512.mask.pmovus.wb.mem.256(i8* %ptr, <16 x i16> %x1, i16 %x2) + ret void +} + +declare <4 x i32> @llvm.x86.avx512.mask.pmaddw.d.128(<8 x i16>, <8 x i16>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_pmaddw_d_128(<8 x i16> %x0, <8 x i16> %x1, <4 x i32> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmaddw_d_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmaddwd %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.pmaddw.d.128(<8 x i16> %x0, <8 x i16> %x1, <4 x i32> %x2, i8 %x3) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmaddw.d.128(<8 x i16> %x0, <8 x i16> %x1, <4 x i32> %x2, i8 -1) + %res2 = add <4 x i32> %res, %res1 + ret <4 x i32> %res2 +} + +declare <8 x i32> @llvm.x86.avx512.mask.pmaddw.d.256(<16 x i16>, <16 x i16>, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_mask_pmaddw_d_256(<16 x i16> %x0, <16 x i16> %x1, <8 x i32> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmaddw_d_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmaddwd %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.mask.pmaddw.d.256(<16 x i16> %x0, <16 x i16> %x1, <8 x i32> %x2, i8 %x3) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmaddw.d.256(<16 x i16> %x0, <16 x i16> %x1, <8 x i32> %x2, i8 -1) + %res2 = add <8 x i32> %res, %res1 + ret <8 x i32> %res2 +} + +declare <8 x i16> @llvm.x86.avx512.mask.pmaddubs.w.128(<16 x i8>, <16 x i8>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_pmaddubs_w_128(<16 x i8> %x0, <16 x i8> %x1, <8 x i16> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmaddubs_w_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddw %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <8 x i16> @llvm.x86.avx512.mask.pmaddubs.w.128(<16 x i8> %x0, <16 x i8> %x1, <8 x i16> %x2, i8 %x3) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmaddubs.w.128(<16 x i8> %x0, <16 x i8> %x1, <8 x i16> %x2, i8 -1) + %res2 = add <8 x i16> %res, %res1 + ret <8 x i16> %res2 +} + +declare <16 x i16> @llvm.x86.avx512.mask.pmaddubs.w.256(<32 x i8>, <32 x i8>, <16 x i16>, i16) + +define <16 x i16>@test_int_x86_avx512_mask_pmaddubs_w_256(<32 x i8> %x0, <32 x i8> %x1, <16 x i16> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmaddubs_w_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddw %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: retq + %res = call <16 x i16> @llvm.x86.avx512.mask.pmaddubs.w.256(<32 x i8> %x0, <32 x i8> %x1, <16 x i16> %x2, i16 %x3) + %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmaddubs.w.256(<32 x i8> %x0, <32 x i8> %x1, <16 x i16> %x2, i16 -1) + %res2 = add <16 x i16> %res, %res1 + ret <16 x i16> %res2 +} + +declare <16 x i8> @llvm.x86.avx512.mask.punpckhb.w.128(<16 x i8>, <16 x i8>, <16 x i8>, i16) + +define <16 x i8>@test_int_x86_avx512_mask_punpckhb_w_128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpckhb_w_128: +; CHECK: vpunpckhbw %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: ## xmm2 = xmm2[8],k1[8],xmm2[9],k1[9],xmm2[10],k1[10],xmm2[11],k1[11],xmm2[12],k1[12],xmm2[13],k1[13],xmm2[14],k1[14],xmm2[15],k1[15] +; CHECK-NEXT: vpunpckhbw %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7d,0x08,0x68,0xc1] +; CHECK-NEXT: ## xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] + %res = call <16 x i8> @llvm.x86.avx512.mask.punpckhb.w.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.punpckhb.w.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 -1) + %res2 = add <16 x i8> %res, %res1 + ret <16 x i8> %res2 +} + +declare <16 x i8> @llvm.x86.avx512.mask.punpcklb.w.128(<16 x i8>, <16 x i8>, <16 x i8>, i16) + +define <16 x i8>@test_int_x86_avx512_mask_punpcklb_w_128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpcklb_w_128: +; CHECK: vpunpcklbw %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: ## xmm2 = xmm2[0],k1[0],xmm2[1],k1[1],xmm2[2],k1[2],xmm2[3],k1[3],xmm2[4],k1[4],xmm2[5],k1[5],xmm2[6],k1[6],xmm2[7],k1[7] +; CHECK-NEXT: vpunpcklbw %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7d,0x08,0x60,0xc1] +; CHECK-NEXT: ## xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] + %res = call <16 x i8> @llvm.x86.avx512.mask.punpcklb.w.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.punpcklb.w.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 -1) + %res2 = add <16 x i8> %res, %res1 + ret <16 x i8> %res2 +} + +declare <32 x i8> @llvm.x86.avx512.mask.punpckhb.w.256(<32 x i8>, <32 x i8>, <32 x i8>, i32) + +define <32 x i8>@test_int_x86_avx512_mask_punpckhb_w_256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpckhb_w_256: +; CHECK: vpunpckhbw %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm2[8],k1[8],ymm2[9],k1[9],ymm2[10],k1[10],ymm2[11],k1[11],ymm2[12],k1[12],ymm2[13],k1[13],ymm2[14],k1[14],ymm2[15],k1[15],ymm2[24],k1[24],ymm2[25],k1[25],ymm2[26],k1[26],ymm2[27],k1[27],ymm2[28],k1[28],ymm2[29],k1[29],ymm2[30],k1[30],ymm2[31],k1[31] +; CHECK-NEXT: vpunpckhbw %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7d,0x28,0x68,0xc1] +; CHECK-NEXT: ## ymm0 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] + %res = call <32 x i8> @llvm.x86.avx512.mask.punpckhb.w.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3) + %res1 = call <32 x i8> @llvm.x86.avx512.mask.punpckhb.w.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 -1) + %res2 = add <32 x i8> %res, %res1 + ret <32 x i8> %res2 +} + +declare <32 x i8> @llvm.x86.avx512.mask.punpcklb.w.256(<32 x i8>, <32 x i8>, <32 x i8>, i32) + +define <32 x i8>@test_int_x86_avx512_mask_punpcklb_w_256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpcklb_w_256: +; CHECK: vpunpcklbw %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm2[0],k1[0],ymm2[1],k1[1],ymm2[2],k1[2],ymm2[3],k1[3],ymm2[4],k1[4],ymm2[5],k1[5],ymm2[6],k1[6],ymm2[7],k1[7],ymm2[16],k1[16],ymm2[17],k1[17],ymm2[18],k1[18],ymm2[19],k1[19],ymm2[20],k1[20],ymm2[21],k1[21],ymm2[22],k1[22],ymm2[23],k1[23] +; CHECK-NEXT: vpunpcklbw %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7d,0x28,0x60,0xc1] +; CHECK-NEXT: ## ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23] + %res = call <32 x i8> @llvm.x86.avx512.mask.punpcklb.w.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3) + %res1 = call <32 x i8> @llvm.x86.avx512.mask.punpcklb.w.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 -1) + %res2 = add <32 x i8> %res, %res1 + ret <32 x i8> %res2 +} + +declare <8 x i16> @llvm.x86.avx512.mask.punpcklw.d.128(<8 x i16>, <8 x i16>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_punpcklw_d_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpcklw_d_128: +; CHECK: vpunpcklwd %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: ## xmm2 = xmm2[0],k1[0],xmm2[1],k1[1],xmm2[2],k1[2],xmm2[3],k1[3] +; CHECK-NEXT: vpunpcklwd %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7d,0x08,0x61,0xc1] +; CHECK-NEXT: ## xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] + %res = call <8 x i16> @llvm.x86.avx512.mask.punpcklw.d.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.punpcklw.d.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1) + %res2 = add <8 x i16> %res, %res1 + ret <8 x i16> %res2 +} + +declare <8 x i16> @llvm.x86.avx512.mask.punpckhw.d.128(<8 x i16>, <8 x i16>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_punpckhw_d_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpckhw_d_128: +; CHECK: vpunpckhwd %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: ## xmm2 = xmm2[4],k1[4],xmm2[5],k1[5],xmm2[6],k1[6],xmm2[7],k1[7] +; CHECK-NEXT: vpunpckhwd %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7d,0x08,0x69,0xc1] +; CHECK-NEXT: ## xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] + %res = call <8 x i16> @llvm.x86.avx512.mask.punpckhw.d.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.punpckhw.d.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1) + %res2 = add <8 x i16> %res, %res1 + ret <8 x i16> %res2 +} + +declare <16 x i16> @llvm.x86.avx512.mask.punpcklw.d.256(<16 x i16>, <16 x i16>, <16 x i16>, i16) + +define <16 x i16>@test_int_x86_avx512_mask_punpcklw_d_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpcklw_d_256: +; CHECK: vpunpcklwd %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm2[0],k1[0],ymm2[1],k1[1],ymm2[2],k1[2],ymm2[3],k1[3],ymm2[8],k1[8],ymm2[9],k1[9],ymm2[10],k1[10],ymm2[11],k1[11] +; CHECK-NEXT: vpunpcklwd %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7d,0x28,0x61,0xc1] +; CHECK-NEXT: ## ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11] + %res = call <16 x i16> @llvm.x86.avx512.mask.punpcklw.d.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) + %res1 = call <16 x i16> @llvm.x86.avx512.mask.punpcklw.d.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1) + %res2 = add <16 x i16> %res, %res1 + ret <16 x i16> %res2 +} + +declare <16 x i16> @llvm.x86.avx512.mask.punpckhw.d.256(<16 x i16>, <16 x i16>, <16 x i16>, i16) + +define <16 x i16>@test_int_x86_avx512_mask_punpckhw_d_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpckhw_d_256: +; CHECK: vpunpckhwd %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm2[4],k1[4],ymm2[5],k1[5],ymm2[6],k1[6],ymm2[7],k1[7],ymm2[12],k1[12],ymm2[13],k1[13],ymm2[14],k1[14],ymm2[15],k1[15] +; CHECK-NEXT: vpunpckhwd %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7d,0x28,0x69,0xc1] +; CHECK-NEXT: ## ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15] + %res = call <16 x i16> @llvm.x86.avx512.mask.punpckhw.d.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) + %res1 = call <16 x i16> @llvm.x86.avx512.mask.punpckhw.d.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1) + %res2 = add <16 x i16> %res, %res1 + ret <16 x i16> %res2 +} + +declare <16 x i8> @llvm.x86.avx512.mask.palignr.128(<16 x i8>, <16 x i8>, i32, <16 x i8>, i16) + +define <16 x i8>@test_int_x86_avx512_mask_palignr_128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x3, i16 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_palignr_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpalignr $2, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vpalignr $2, %xmm1, %xmm0, %xmm3 {%k1} {z} +; CHECK-NEXT: vpalignr $2, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddb %xmm3, %xmm2, %xmm1 +; CHECK-NEXT: vpaddb %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <16 x i8> @llvm.x86.avx512.mask.palignr.128(<16 x i8> %x0, <16 x i8> %x1, i32 2, <16 x i8> %x3, i16 %x4) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.palignr.128(<16 x i8> %x0, <16 x i8> %x1, i32 2, <16 x i8> zeroinitializer, i16 %x4) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.palignr.128(<16 x i8> %x0, <16 x i8> %x1, i32 2, <16 x i8> %x3, i16 -1) + %res3 = add <16 x i8> %res, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare <32 x i8> @llvm.x86.avx512.mask.palignr.256(<32 x i8>, <32 x i8>, i32, <32 x i8>, i32) + +define <32 x i8>@test_int_x86_avx512_mask_palignr_256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x3, i32 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_palignr_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovd %edi, %k1 +; CHECK-NEXT: vpalignr $2, %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: vpalignr $2, %ymm1, %ymm0, %ymm3 {%k1} {z} +; CHECK-NEXT: vpalignr $2, %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddb %ymm3, %ymm2, %ymm1 +; CHECK-NEXT: vpaddb %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <32 x i8> @llvm.x86.avx512.mask.palignr.256(<32 x i8> %x0, <32 x i8> %x1, i32 2, <32 x i8> %x3, i32 %x4) + %res1 = call <32 x i8> @llvm.x86.avx512.mask.palignr.256(<32 x i8> %x0, <32 x i8> %x1, i32 2, <32 x i8> zeroinitializer, i32 %x4) + %res2 = call <32 x i8> @llvm.x86.avx512.mask.palignr.256(<32 x i8> %x0, <32 x i8> %x1, i32 2, <32 x i8> %x3, i32 -1) + %res3 = add <32 x i8> %res, %res1 + %res4 = add <32 x i8> %res3, %res2 + ret <32 x i8> %res4 +} + +declare <8 x i16> @llvm.x86.avx512.mask.dbpsadbw.128(<16 x i8>, <16 x i8>, i32, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_dbpsadbw_128(<16 x i8> %x0, <16 x i8> %x1, <8 x i16> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_dbpsadbw_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vdbpsadbw $2, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vdbpsadbw $2, %xmm1, %xmm0, %xmm3 {%k1} {z} +; CHECK-NEXT: vdbpsadbw $2, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddw %xmm3, %xmm2, %xmm1 +; CHECK-NEXT: vpaddw %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <8 x i16> @llvm.x86.avx512.mask.dbpsadbw.128(<16 x i8> %x0, <16 x i8> %x1, i32 2, <8 x i16> %x3, i8 %x4) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.dbpsadbw.128(<16 x i8> %x0, <16 x i8> %x1, i32 2, <8 x i16> zeroinitializer, i8 %x4) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.dbpsadbw.128(<16 x i8> %x0, <16 x i8> %x1, i32 2, <8 x i16> %x3, i8 -1) + %res3 = add <8 x i16> %res, %res1 + %res4 = add <8 x i16> %res2, %res3 + ret <8 x i16> %res4 +} + +declare <16 x i16> @llvm.x86.avx512.mask.dbpsadbw.256(<32 x i8>, <32 x i8>, i32, <16 x i16>, i16) + +define <16 x i16>@test_int_x86_avx512_mask_dbpsadbw_256(<32 x i8> %x0, <32 x i8> %x1, <16 x i16> %x3, i16 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_dbpsadbw_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vdbpsadbw $2, %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: vdbpsadbw $2, %ymm1, %ymm0, %ymm3 {%k1} {z} +; CHECK-NEXT: vdbpsadbw $2, %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddw %ymm3, %ymm2, %ymm1 +; CHECK-NEXT: vpaddw %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <16 x i16> @llvm.x86.avx512.mask.dbpsadbw.256(<32 x i8> %x0, <32 x i8> %x1, i32 2, <16 x i16> %x3, i16 %x4) + %res1 = call <16 x i16> @llvm.x86.avx512.mask.dbpsadbw.256(<32 x i8> %x0, <32 x i8> %x1, i32 2, <16 x i16> zeroinitializer, i16 %x4) + %res2 = call <16 x i16> @llvm.x86.avx512.mask.dbpsadbw.256(<32 x i8> %x0, <32 x i8> %x1, i32 2, <16 x i16> %x3, i16 -1) + %res3 = add <16 x i16> %res, %res1 + %res4 = add <16 x i16> %res3, %res2 + ret <16 x i16> %res4 +} + +declare <32 x i8> @llvm.x86.avx512.pbroadcastb.256(<16 x i8>, <32 x i8>, i32) + +define <32 x i8>@test_int_x86_avx512_pbroadcastb_256(<16 x i8> %x0, <32 x i8> %x1, i32 %mask) { +; CHECK-LABEL: test_int_x86_avx512_pbroadcastb_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovd %edi, %k1 +; CHECK-NEXT: vpbroadcastb %xmm0, %ymm1 {%k1} +; CHECK-NEXT: vpbroadcastb %xmm0, %ymm2 {%k1} {z} +; CHECK-NEXT: vpbroadcastb %xmm0, %ymm0 +; CHECK-NEXT: vpaddb %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddb %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: retq + %res = call <32 x i8> @llvm.x86.avx512.pbroadcastb.256(<16 x i8> %x0, <32 x i8> %x1, i32 -1) + %res1 = call <32 x i8> @llvm.x86.avx512.pbroadcastb.256(<16 x i8> %x0, <32 x i8> %x1, i32 %mask) + %res2 = call <32 x i8> @llvm.x86.avx512.pbroadcastb.256(<16 x i8> %x0, <32 x i8> zeroinitializer, i32 %mask) + %res3 = add <32 x i8> %res, %res1 + %res4 = add <32 x i8> %res2, %res3 + ret <32 x i8> %res4 +} + +declare <16 x i8> @llvm.x86.avx512.pbroadcastb.128(<16 x i8>, <16 x i8>, i16) + +define <16 x i8>@test_int_x86_avx512_pbroadcastb_128(<16 x i8> %x0, <16 x i8> %x1, i16 %mask) { +; CHECK-LABEL: test_int_x86_avx512_pbroadcastb_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpbroadcastb %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpbroadcastb %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpbroadcastb %xmm0, %xmm0 +; CHECK-NEXT: vpaddb %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddb %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <16 x i8> @llvm.x86.avx512.pbroadcastb.128(<16 x i8> %x0, <16 x i8> %x1, i16 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.pbroadcastb.128(<16 x i8> %x0, <16 x i8> %x1, i16 %mask) + %res2 = call <16 x i8> @llvm.x86.avx512.pbroadcastb.128(<16 x i8> %x0, <16 x i8> zeroinitializer, i16 %mask) + %res3 = add <16 x i8> %res, %res1 + %res4 = add <16 x i8> %res2, %res3 + ret <16 x i8> %res4 +} + +declare <16 x i16> @llvm.x86.avx512.pbroadcastw.256(<8 x i16>, <16 x i16>, i16) + +define <16 x i16>@test_int_x86_avx512_pbroadcastw_256(<8 x i16> %x0, <16 x i16> %x1, i16 %mask) { +; CHECK-LABEL: test_int_x86_avx512_pbroadcastw_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpbroadcastw %xmm0, %ymm1 {%k1} +; CHECK-NEXT: vpbroadcastw %xmm0, %ymm2 {%k1} {z} +; CHECK-NEXT: vpbroadcastw %xmm0, %ymm0 +; CHECK-NEXT: vpaddw %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddw %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: retq + %res = call <16 x i16> @llvm.x86.avx512.pbroadcastw.256(<8 x i16> %x0, <16 x i16> %x1, i16 -1) + %res1 = call <16 x i16> @llvm.x86.avx512.pbroadcastw.256(<8 x i16> %x0, <16 x i16> %x1, i16 %mask) + %res2 = call <16 x i16> @llvm.x86.avx512.pbroadcastw.256(<8 x i16> %x0, <16 x i16> zeroinitializer, i16 %mask) + %res3 = add <16 x i16> %res, %res1 + %res4 = add <16 x i16> %res2, %res3 + ret <16 x i16> %res4 +} + +declare <8 x i16> @llvm.x86.avx512.pbroadcastw.128(<8 x i16>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_pbroadcastw_128(<8 x i16> %x0, <8 x i16> %x1, i8 %mask) { +; CHECK-LABEL: test_int_x86_avx512_pbroadcastw_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpbroadcastw %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpbroadcastw %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpbroadcastw %xmm0, %xmm0 +; CHECK-NEXT: vpaddw %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddw %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <8 x i16> @llvm.x86.avx512.pbroadcastw.128(<8 x i16> %x0, <8 x i16> %x1, i8 -1) + %res1 = call <8 x i16> @llvm.x86.avx512.pbroadcastw.128(<8 x i16> %x0, <8 x i16> %x1, i8 %mask) + %res2 = call <8 x i16> @llvm.x86.avx512.pbroadcastw.128(<8 x i16> %x0, <8 x i16> zeroinitializer, i8 %mask) + %res3 = add <8 x i16> %res, %res1 + %res4 = add <8 x i16> %res2, %res3 + ret <8 x i16> %res4 +} + +declare <64 x i8> @llvm.x86.avx512.pbroadcastb.512(<16 x i8>, <64 x i8>, i64) + +define <64 x i8>@test_int_x86_avx512_pbroadcastb_512(<16 x i8> %x0, <64 x i8> %x1, i64 %mask) { +; CHECK-LABEL: test_int_x86_avx512_pbroadcastb_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovq %rdi, %k1 ## encoding: [0xc4,0xe1,0xfb,0x92,0xcf] +; CHECK-NEXT: vpbroadcastb %xmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x78,0xc8] +; CHECK-NEXT: vpbroadcastb %xmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x78,0xd0] +; CHECK-NEXT: vpbroadcastb %xmm0, %zmm0 ## encoding: [0x62,0xf2,0x7d,0x48,0x78,0xc0] +; CHECK-NEXT: vpaddb %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfc,0xc1] +; CHECK-NEXT: vpaddb %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfc,0xc0] +; CHECK-NEXT: retq ## encoding: [0xc3] + %res = call <64 x i8> @llvm.x86.avx512.pbroadcastb.512(<16 x i8> %x0, <64 x i8> %x1, i64 -1) + %res1 = call <64 x i8> @llvm.x86.avx512.pbroadcastb.512(<16 x i8> %x0, <64 x i8> %x1, i64 %mask) + %res2 = call <64 x i8> @llvm.x86.avx512.pbroadcastb.512(<16 x i8> %x0, <64 x i8> zeroinitializer, i64 %mask) + %res3 = add <64 x i8> %res, %res1 + %res4 = add <64 x i8> %res2, %res3 + ret <64 x i8> %res4 +} + +declare <32 x i16> @llvm.x86.avx512.pbroadcastw.512(<8 x i16>, <32 x i16>, i32) + +define <32 x i16>@test_int_x86_avx512_pbroadcastw_512(<8 x i16> %x0, <32 x i16> %x1, i32 %mask) { +; CHECK-LABEL: test_int_x86_avx512_pbroadcastw_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf] +; CHECK-NEXT: vpbroadcastw %xmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x79,0xc8] +; CHECK-NEXT: vpbroadcastw %xmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x79,0xd0] +; CHECK-NEXT: vpbroadcastw %xmm0, %zmm0 ## encoding: [0x62,0xf2,0x7d,0x48,0x79,0xc0] +; CHECK-NEXT: vpaddw %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc1] +; CHECK-NEXT: vpaddw %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0] +; CHECK-NEXT: retq ## encoding: [0xc3] + %res = call <32 x i16> @llvm.x86.avx512.pbroadcastw.512(<8 x i16> %x0, <32 x i16> %x1, i32 -1) + %res1 = call <32 x i16> @llvm.x86.avx512.pbroadcastw.512(<8 x i16> %x0, <32 x i16> %x1, i32 %mask) + %res2 = call <32 x i16> @llvm.x86.avx512.pbroadcastw.512(<8 x i16> %x0, <32 x i16> zeroinitializer, i32 %mask) + %res3 = add <32 x i16> %res, %res1 + %res4 = add <32 x i16> %res2, %res3 + ret <32 x i16> %res4 +} + +declare i16 @llvm.x86.avx512.cvtb2mask.128(<16 x i8>) + +define i16@test_int_x86_avx512_cvtb2mask_128(<16 x i8> %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtb2mask_128: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovb2m %xmm0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq + %res = call i16 @llvm.x86.avx512.cvtb2mask.128(<16 x i8> %x0) + ret i16 %res +} + +declare i32 @llvm.x86.avx512.cvtb2mask.256(<32 x i8>) + +define i32@test_int_x86_avx512_cvtb2mask_256(<32 x i8> %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtb2mask_256: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovb2m %ymm0, %k0 +; CHECK-NEXT: kmovd %k0, %eax +; CHECK-NEXT: retq + %res = call i32 @llvm.x86.avx512.cvtb2mask.256(<32 x i8> %x0) + ret i32 %res +} + +declare i8 @llvm.x86.avx512.cvtw2mask.128(<8 x i16>) + +define i8@test_int_x86_avx512_cvtw2mask_128(<8 x i16> %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtw2mask_128: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovw2m %xmm0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq + %res = call i8 @llvm.x86.avx512.cvtw2mask.128(<8 x i16> %x0) + ret i8 %res +} + +declare i16 @llvm.x86.avx512.cvtw2mask.256(<16 x i16>) + +define i16@test_int_x86_avx512_cvtw2mask_256(<16 x i16> %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtw2mask_256: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovw2m %ymm0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq + %res = call i16 @llvm.x86.avx512.cvtw2mask.256(<16 x i16> %x0) + ret i16 %res +} + +declare <16 x i8> @llvm.x86.avx512.cvtmask2b.128(i16) + +define <16 x i8>@test_int_x86_avx512_cvtmask2b_128(i16 %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtmask2b_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k0 +; CHECK-NEXT: vpmovm2b %k0, %xmm0 +; CHECK-NEXT: retq + %res = call <16 x i8> @llvm.x86.avx512.cvtmask2b.128(i16 %x0) + ret <16 x i8> %res +} + +declare <32 x i8> @llvm.x86.avx512.cvtmask2b.256(i32) + +define <32 x i8>@test_int_x86_avx512_cvtmask2b_256(i32 %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtmask2b_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovd %edi, %k0 +; CHECK-NEXT: vpmovm2b %k0, %ymm0 +; CHECK-NEXT: retq + %res = call <32 x i8> @llvm.x86.avx512.cvtmask2b.256(i32 %x0) + ret <32 x i8> %res +} + +declare <8 x i16> @llvm.x86.avx512.cvtmask2w.128(i8) + +define <8 x i16>@test_int_x86_avx512_cvtmask2w_128(i8 %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtmask2w_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k0 +; CHECK-NEXT: vpmovm2w %k0, %xmm0 +; CHECK-NEXT: retq + %res = call <8 x i16> @llvm.x86.avx512.cvtmask2w.128(i8 %x0) + ret <8 x i16> %res +} + +declare <16 x i16> @llvm.x86.avx512.cvtmask2w.256(i16) + +define <16 x i16>@test_int_x86_avx512_cvtmask2w_256(i16 %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtmask2w_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k0 +; CHECK-NEXT: vpmovm2w %k0, %ymm0 +; CHECK-NEXT: retq + %res = call <16 x i16> @llvm.x86.avx512.cvtmask2w.256(i16 %x0) + ret <16 x i16> %res +} + +declare <8 x i16> @llvm.x86.avx512.mask.psrl.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_psrl_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_psrl_w_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsrlw %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vpsrlw %xmm1, %xmm0, %xmm3 {%k1} {z} +; CHECK-NEXT: vpsrlw %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddw %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: vpaddw %xmm0, %xmm3, %xmm0 +; CHECK-NEXT: retq + %res = call <8 x i16> @llvm.x86.avx512.mask.psrl.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.psrl.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.psrl.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3) + %res3 = add <8 x i16> %res, %res1 + %res4 = add <8 x i16> %res2, %res3 + ret <8 x i16> %res4 +} + +declare <16 x i16> @llvm.x86.avx512.mask.psrl.w.256(<16 x i16>, <8 x i16>, <16 x i16>, i16) + +define <16 x i16>@test_int_x86_avx512_mask_psrl_w_256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_psrl_w_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vpsrlw %xmm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: vpsrlw %xmm1, %ymm0, %ymm3 {%k1} {z} +; CHECK-NEXT: vpsrlw %xmm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddw %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: vpaddw %ymm3, %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <16 x i16> @llvm.x86.avx512.mask.psrl.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2, i16 %x3) + %res1 = call <16 x i16> @llvm.x86.avx512.mask.psrl.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2, i16 -1) + %res2 = call <16 x i16> @llvm.x86.avx512.mask.psrl.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3) + %res3 = add <16 x i16> %res, %res1 + %res4 = add <16 x i16> %res3, %res2 + ret <16 x i16> %res4 +} + +declare <8 x i16> @llvm.x86.avx512.mask.psrl.wi.128(<8 x i16>, i8, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_psrl_wi_128(<8 x i16> %x0, i8 %x1, <8 x i16> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_psrl_wi_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %sil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpsrlw $3, %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpsrlw $3, %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpsrlw $3, %xmm0, %xmm0 +; CHECK-NEXT: vpaddw %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: vpaddw %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <8 x i16> @llvm.x86.avx512.mask.psrl.wi.128(<8 x i16> %x0, i8 3, <8 x i16> %x2, i8 %x3) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.psrl.wi.128(<8 x i16> %x0, i8 3, <8 x i16> %x2, i8 -1) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.psrl.wi.128(<8 x i16> %x0, i8 3, <8 x i16> zeroinitializer, i8 %x3) + %res3 = add <8 x i16> %res, %res1 + %res4 = add <8 x i16> %res2, %res3 + ret <8 x i16> %res4 +} + +declare <16 x i16> @llvm.x86.avx512.mask.psrl.wi.256(<16 x i16>, i8, <16 x i16>, i16) + +define <16 x i16>@test_int_x86_avx512_mask_psrl_wi_256(<16 x i16> %x0, i8 %x1, <16 x i16> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_psrl_wi_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %esi, %k1 +; CHECK-NEXT: vpsrlw $3, %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vpsrlw $3, %ymm0, %ymm2 {%k1} {z} +; CHECK-NEXT: vpsrlw $3, %ymm0, %ymm0 +; CHECK-NEXT: vpaddw %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: vpaddw %ymm2, %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <16 x i16> @llvm.x86.avx512.mask.psrl.wi.256(<16 x i16> %x0, i8 3, <16 x i16> %x2, i16 %x3) + %res1 = call <16 x i16> @llvm.x86.avx512.mask.psrl.wi.256(<16 x i16> %x0, i8 3, <16 x i16> %x2, i16 -1) + %res2 = call <16 x i16> @llvm.x86.avx512.mask.psrl.wi.256(<16 x i16> %x0, i8 3, <16 x i16> zeroinitializer, i16 %x3) + %res3 = add <16 x i16> %res, %res1 + %res4 = add <16 x i16> %res3, %res2 + ret <16 x i16> %res4 +} diff --git a/test/CodeGen/X86/avx512cd-intrinsics.ll b/test/CodeGen/X86/avx512cd-intrinsics.ll new file mode 100644 index 000000000000..29f17bbc0190 --- /dev/null +++ b/test/CodeGen/X86/avx512cd-intrinsics.ll @@ -0,0 +1,18 @@ +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512cd | FileCheck %s + +define <16 x i32> @test_x86_vbroadcastmw_512(i16 %a0) { + ; CHECK: test_x86_vbroadcastmw_512 + ; CHECK: vpbroadcastmw2d %k0, %zmm0 + %res = call <16 x i32> @llvm.x86.avx512.broadcastmw.512(i16 %a0) ; + ret <16 x i32> %res +} +declare <16 x i32> @llvm.x86.avx512.broadcastmw.512(i16) + +define <8 x i64> @test_x86_broadcastmb_512(i8 %a0) { + ; CHECK: test_x86_broadcastmb_512 + ; CHECK: vpbroadcastmb2q %k0, %zmm0 + %res = call <8 x i64> @llvm.x86.avx512.broadcastmb.512(i8 %a0) ; + ret <8 x i64> %res +} +declare <8 x i64> @llvm.x86.avx512.broadcastmb.512(i8) + diff --git a/test/CodeGen/X86/avx512cdvl-intrinsics.ll b/test/CodeGen/X86/avx512cdvl-intrinsics.ll new file mode 100644 index 000000000000..14e91e1a8768 --- /dev/null +++ b/test/CodeGen/X86/avx512cdvl-intrinsics.ll @@ -0,0 +1,179 @@ +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512cd -mattr=+avx512vl| FileCheck %s + +declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>, i1) nounwind readonly + +declare <4 x i32> @llvm.x86.avx512.mask.lzcnt.d.128(<4 x i32>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_vplzcnt_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_vplzcnt_d_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vplzcntd %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vplzcntd %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vplzcntd %xmm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: vpaddd %xmm2, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.lzcnt.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.lzcnt.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 -1) + %res3 = call <4 x i32> @llvm.x86.avx512.mask.lzcnt.d.128(<4 x i32> %x0, <4 x i32> zeroinitializer, i8 %x2) + %res2 = add <4 x i32> %res, %res1 + %res4 = add <4 x i32> %res2, %res3 + ret <4 x i32> %res4 +} + +declare <8 x i32> @llvm.x86.avx512.mask.lzcnt.d.256(<8 x i32>, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_mask_vplzcnt_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_vplzcnt_d_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vplzcntd %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vplzcntd %ymm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.mask.lzcnt.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.lzcnt.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 -1) + %res2 = add <8 x i32> %res, %res1 + ret <8 x i32> %res2 +} + +declare <2 x i64> @llvm.x86.avx512.mask.lzcnt.q.128(<2 x i64>, <2 x i64>, i8) + +define <2 x i64>@test_int_x86_avx512_mask_vplzcnt_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_vplzcnt_q_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vplzcntq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vplzcntq %xmm0, %xmm0 +; CHECK-NEXT: vpaddq %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x i64> @llvm.x86.avx512.mask.lzcnt.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) + %res1 = call <2 x i64> @llvm.x86.avx512.mask.lzcnt.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 -1) + %res2 = add <2 x i64> %res, %res1 + ret <2 x i64> %res2 +} + +declare <4 x i64> @llvm.x86.avx512.mask.lzcnt.q.256(<4 x i64>, <4 x i64>, i8) + +define <4 x i64>@test_int_x86_avx512_mask_vplzcnt_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_vplzcnt_q_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vplzcntq %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vplzcntq %ymm0, %ymm0 +; CHECK-NEXT: vpaddq %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.mask.lzcnt.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) + %res1 = call <4 x i64> @llvm.x86.avx512.mask.lzcnt.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 -1) + %res2 = add <4 x i64> %res, %res1 + ret <4 x i64> %res2 +} + +declare <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_vpconflict_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpconflict_d_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpconflictd %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpconflictd %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpconflictd %xmm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: vpaddd %xmm2, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 -1) + %res3 = call <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32> %x0, <4 x i32> zeroinitializer, i8 %x2) + %res2 = add <4 x i32> %res, %res1 + %res4 = add <4 x i32> %res2, %res3 + ret <4 x i32> %res4 +} + +declare <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32>, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_mask_vpconflict_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpconflict_d_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpconflictd %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vpconflictd %ymm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 -1) + %res2 = add <8 x i32> %res, %res1 + ret <8 x i32> %res2 +} + +declare <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64>, <2 x i64>, i8) + +define <2 x i64>@test_int_x86_avx512_mask_vpconflict_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpconflict_q_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpconflictq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpconflictq %xmm0, %xmm0 +; CHECK-NEXT: vpaddq %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) + %res1 = call <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 -1) + %res2 = add <2 x i64> %res, %res1 + ret <2 x i64> %res2 +} + +declare <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64>, <4 x i64>, i8) + +define <4 x i64>@test_int_x86_avx512_mask_vpconflict_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpconflict_q_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpconflictq %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vpconflictq %ymm0, %ymm0 +; CHECK-NEXT: vpaddq %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) + %res1 = call <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 -1) + %res2 = add <4 x i64> %res, %res1 + ret <4 x i64> %res2 +} + +define <8 x i32> @test_x86_vbroadcastmw_256(i16 %a0) { + ; CHECK: test_x86_vbroadcastmw_256 + ; CHECK: vpbroadcastmw2d %k0, %ymm0 + %res = call <8 x i32> @llvm.x86.avx512.broadcastmw.256(i16 %a0) ; + ret <8 x i32> %res +} +declare <8 x i32> @llvm.x86.avx512.broadcastmw.256(i16) + +define <4 x i32> @test_x86_vbroadcastmw_128(i16 %a0) { + ; CHECK: test_x86_vbroadcastmw_128 + ; CHECK: vpbroadcastmw2d %k0, %xmm0 + %res = call <4 x i32> @llvm.x86.avx512.broadcastmw.128(i16 %a0) ; + ret <4 x i32> %res +} +declare <4 x i32> @llvm.x86.avx512.broadcastmw.128(i16) + +define <4 x i64> @test_x86_broadcastmb_256(i8 %a0) { + ; CHECK: test_x86_broadcastmb_256 + ; CHECK: vpbroadcastmb2q %k0, %ymm0 + %res = call <4 x i64> @llvm.x86.avx512.broadcastmb.256(i8 %a0) ; + ret <4 x i64> %res +} +declare <4 x i64> @llvm.x86.avx512.broadcastmb.256(i8) + +define <2 x i64> @test_x86_broadcastmb_128(i8 %a0) { + ; CHECK: test_x86_broadcastmb_128 + ; CHECK: vpbroadcastmb2q %k0, %xmm0 + %res = call <2 x i64> @llvm.x86.avx512.broadcastmb.128(i8 %a0) ; + ret <2 x i64> %res +} +declare <2 x i64> @llvm.x86.avx512.broadcastmb.128(i8) diff --git a/test/CodeGen/X86/avx512dq-intrinsics.ll b/test/CodeGen/X86/avx512dq-intrinsics.ll new file mode 100644 index 000000000000..a59fe393f556 --- /dev/null +++ b/test/CodeGen/X86/avx512dq-intrinsics.ll @@ -0,0 +1,667 @@ + +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512dq | FileCheck %s + +declare <8 x i64> @llvm.x86.avx512.mask.cvtpd2qq.512(<8 x double>, <8 x i64>, i8, i32) + +define <8 x i64>@test_int_x86_avx512_mask_cvt_pd2qq_512(<8 x double> %x0, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2qq_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtpd2qq {ru-sae}, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vcvtpd2qq {rn-sae}, %zmm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.cvtpd2qq.512(<8 x double> %x0, <8 x i64> %x1, i8 %x2, i32 2) + %res1 = call <8 x i64> @llvm.x86.avx512.mask.cvtpd2qq.512(<8 x double> %x0, <8 x i64> %x1, i8 -1, i32 0) + %res2 = add <8 x i64> %res, %res1 + ret <8 x i64> %res2 +} + +declare <8 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.512(<8 x double>, <8 x i64>, i8, i32) + +define <8 x i64>@test_int_x86_avx512_mask_cvt_pd2uqq_512(<8 x double> %x0, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2uqq_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtpd2uqq {ru-sae}, %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vcvtpd2uqq {rn-sae}, %zmm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.512(<8 x double> %x0, <8 x i64> %x1, i8 %x2, i32 2) + %res1 = call <8 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.512(<8 x double> %x0, <8 x i64> %x1, i8 -1, i32 0) + %res2 = add <8 x i64> %res, %res1 + ret <8 x i64> %res2 +} + +declare <8 x i64> @llvm.x86.avx512.mask.cvtps2qq.512(<8 x float>, <8 x i64>, i8, i32) + +define <8 x i64>@test_int_x86_avx512_mask_cvt_ps2qq_512(<8 x float> %x0, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2qq_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtps2qq {ru-sae}, %ymm0, %zmm1 {%k1} +; CHECK-NEXT: vcvtps2qq {rn-sae}, %ymm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.cvtps2qq.512(<8 x float> %x0, <8 x i64> %x1, i8 %x2, i32 2) + %res1 = call <8 x i64> @llvm.x86.avx512.mask.cvtps2qq.512(<8 x float> %x0, <8 x i64> %x1, i8 -1, i32 0) + %res2 = add <8 x i64> %res, %res1 + ret <8 x i64> %res2 +} + +declare <8 x i64> @llvm.x86.avx512.mask.cvtps2uqq.512(<8 x float>, <8 x i64>, i8, i32) + +define <8 x i64>@test_int_x86_avx512_mask_cvt_ps2uqq_512(<8 x float> %x0, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2uqq_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtps2uqq {ru-sae}, %ymm0, %zmm1 {%k1} +; CHECK-NEXT: vcvtps2uqq {rn-sae}, %ymm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.cvtps2uqq.512(<8 x float> %x0, <8 x i64> %x1, i8 %x2, i32 2) + %res1 = call <8 x i64> @llvm.x86.avx512.mask.cvtps2uqq.512(<8 x float> %x0, <8 x i64> %x1, i8 -1, i32 0) + %res2 = add <8 x i64> %res, %res1 + ret <8 x i64> %res2 +} + +declare <8 x double> @llvm.x86.avx512.mask.cvtqq2pd.512(<8 x i64>, <8 x double>, i8, i32) + +define <8 x double>@test_int_x86_avx512_mask_cvt_qq2pd_512(<8 x i64> %x0, <8 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_qq2pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtqq2pd %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vcvtqq2pd {rn-sae}, %zmm0, %zmm0 +; CHECK-NEXT: vaddpd %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.cvtqq2pd.512(<8 x i64> %x0, <8 x double> %x1, i8 %x2, i32 4) + %res1 = call <8 x double> @llvm.x86.avx512.mask.cvtqq2pd.512(<8 x i64> %x0, <8 x double> %x1, i8 -1, i32 0) + %res2 = fadd <8 x double> %res, %res1 + ret <8 x double> %res2 +} + +declare <8 x float> @llvm.x86.avx512.mask.cvtqq2ps.512(<8 x i64>, <8 x float>, i8, i32) + +define <8 x float>@test_int_x86_avx512_mask_cvt_qq2ps_512(<8 x i64> %x0, <8 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_qq2ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtqq2ps %zmm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtqq2ps {rn-sae}, %zmm0, %ymm0 +; CHECK-NEXT: vaddps %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.avx512.mask.cvtqq2ps.512(<8 x i64> %x0, <8 x float> %x1, i8 %x2, i32 4) + %res1 = call <8 x float> @llvm.x86.avx512.mask.cvtqq2ps.512(<8 x i64> %x0, <8 x float> %x1, i8 -1, i32 0) + %res2 = fadd <8 x float> %res, %res1 + ret <8 x float> %res2 +} + +declare <8 x i64> @llvm.x86.avx512.mask.cvttpd2qq.512(<8 x double>, <8 x i64>, i8, i32) + +define <8 x i64>@test_int_x86_avx512_mask_cvtt_pd2qq_512(<8 x double> %x0, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2qq_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvttpd2qq %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vcvttpd2qq {sae}, %zmm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.cvttpd2qq.512(<8 x double> %x0, <8 x i64> %x1, i8 %x2, i32 4) + %res1 = call <8 x i64> @llvm.x86.avx512.mask.cvttpd2qq.512(<8 x double> %x0, <8 x i64> %x1, i8 -1, i32 8) + %res2 = add <8 x i64> %res, %res1 + ret <8 x i64> %res2 +} + +declare <8 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.512(<8 x double>, <8 x i64>, i8, i32) + +define <8 x i64>@test_int_x86_avx512_mask_cvtt_pd2uqq_512(<8 x double> %x0, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2uqq_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvttpd2uqq %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vcvttpd2uqq {sae}, %zmm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.512(<8 x double> %x0, <8 x i64> %x1, i8 %x2, i32 4) + %res1 = call <8 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.512(<8 x double> %x0, <8 x i64> %x1, i8 -1, i32 8) + %res2 = add <8 x i64> %res, %res1 + ret <8 x i64> %res2 +} + +declare <8 x i64> @llvm.x86.avx512.mask.cvttps2qq.512(<8 x float>, <8 x i64>, i8, i32) + +define <8 x i64>@test_int_x86_avx512_mask_cvtt_ps2qq_512(<8 x float> %x0, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2qq_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvttps2qq %ymm0, %zmm1 {%k1} +; CHECK-NEXT: vcvttps2qq {sae}, %ymm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.cvttps2qq.512(<8 x float> %x0, <8 x i64> %x1, i8 %x2, i32 4) + %res1 = call <8 x i64> @llvm.x86.avx512.mask.cvttps2qq.512(<8 x float> %x0, <8 x i64> %x1, i8 -1, i32 8) + %res2 = add <8 x i64> %res, %res1 + ret <8 x i64> %res2 +} + +declare <8 x i64> @llvm.x86.avx512.mask.cvttps2uqq.512(<8 x float>, <8 x i64>, i8, i32) + +define <8 x i64>@test_int_x86_avx512_mask_cvtt_ps2uqq_512(<8 x float> %x0, <8 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2uqq_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvttps2uqq %ymm0, %zmm1 {%k1} +; CHECK-NEXT: vcvttps2uqq {sae}, %ymm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.cvttps2uqq.512(<8 x float> %x0, <8 x i64> %x1, i8 %x2, i32 4) + %res1 = call <8 x i64> @llvm.x86.avx512.mask.cvttps2uqq.512(<8 x float> %x0, <8 x i64> %x1, i8 -1, i32 8) + %res2 = add <8 x i64> %res, %res1 + ret <8 x i64> %res2 +} + +declare <8 x double> @llvm.x86.avx512.mask.cvtuqq2pd.512(<8 x i64>, <8 x double>, i8, i32) + +define <8 x double>@test_int_x86_avx512_mask_cvt_uqq2pd_512(<8 x i64> %x0, <8 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_uqq2pd_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtuqq2pd %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vcvtuqq2pd {rn-sae}, %zmm0, %zmm0 +; CHECK-NEXT: vaddpd %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.cvtuqq2pd.512(<8 x i64> %x0, <8 x double> %x1, i8 %x2, i32 4) + %res1 = call <8 x double> @llvm.x86.avx512.mask.cvtuqq2pd.512(<8 x i64> %x0, <8 x double> %x1, i8 -1, i32 0) + %res2 = fadd <8 x double> %res, %res1 + ret <8 x double> %res2 +} + +declare <8 x float> @llvm.x86.avx512.mask.cvtuqq2ps.512(<8 x i64>, <8 x float>, i8, i32) + +define <8 x float>@test_int_x86_avx512_mask_cvt_uqq2ps_512(<8 x i64> %x0, <8 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_uqq2ps_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtuqq2ps %zmm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtuqq2ps {rn-sae}, %zmm0, %ymm0 +; CHECK-NEXT: vaddps %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.avx512.mask.cvtuqq2ps.512(<8 x i64> %x0, <8 x float> %x1, i8 %x2, i32 4) + %res1 = call <8 x float> @llvm.x86.avx512.mask.cvtuqq2ps.512(<8 x i64> %x0, <8 x float> %x1, i8 -1, i32 0) + %res2 = fadd <8 x float> %res, %res1 + ret <8 x float> %res2 +} + +declare <8 x double> @llvm.x86.avx512.mask.reduce.pd.512(<8 x double>, i32, <8 x double>, i8, i32) +; CHECK-LABEL: @test_int_x86_avx512_mask_reduce_pd_512 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vreducepd {{.*}}{%k1} +; CHECK: vreducepd +; CHECK: {sae} +define <8 x double>@test_int_x86_avx512_mask_reduce_pd_512(<8 x double> %x0, <8 x double> %x2, i8 %x3) { + %res = call <8 x double> @llvm.x86.avx512.mask.reduce.pd.512(<8 x double> %x0, i32 8, <8 x double> %x2, i8 %x3, i32 4) + %res1 = call <8 x double> @llvm.x86.avx512.mask.reduce.pd.512(<8 x double> %x0, i32 4, <8 x double> %x2, i8 -1, i32 8) + %res2 = fadd <8 x double> %res, %res1 + ret <8 x double> %res2 +} + +declare <16 x float> @llvm.x86.avx512.mask.reduce.ps.512(<16 x float>, i32, <16 x float>, i16, i32) +; CHECK-LABEL: @test_int_x86_avx512_mask_reduce_ps_512 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vreduceps +; CHECK: {sae} +; CKECK: {%k1} +; CHECK: vreduceps +define <16 x float>@test_int_x86_avx512_mask_reduce_ps_512(<16 x float> %x0, <16 x float> %x2, i16 %x3) { + %res = call <16 x float> @llvm.x86.avx512.mask.reduce.ps.512(<16 x float> %x0, i32 44, <16 x float> %x2, i16 %x3, i32 8) + %res1 = call <16 x float> @llvm.x86.avx512.mask.reduce.ps.512(<16 x float> %x0, i32 11, <16 x float> %x2, i16 -1, i32 4) + %res2 = fadd <16 x float> %res, %res1 + ret <16 x float> %res2 +} + +declare <8 x double> @llvm.x86.avx512.mask.range.pd.512(<8 x double>, <8 x double>, i32, <8 x double>, i8, i32) +; CHECK-LABEL: @test_int_x86_avx512_mask_range_pd_512 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vrangepd +; CKECK: {%k1} +; CHECK: vrangepd +; CHECK: {sae} +define <8 x double>@test_int_x86_avx512_mask_range_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x3, i8 %x4) { + %res = call <8 x double> @llvm.x86.avx512.mask.range.pd.512(<8 x double> %x0, <8 x double> %x1, i32 8, <8 x double> %x3, i8 %x4, i32 4) + %res1 = call <8 x double> @llvm.x86.avx512.mask.range.pd.512(<8 x double> %x0, <8 x double> %x1, i32 4, <8 x double> %x3, i8 -1, i32 8) + %res2 = fadd <8 x double> %res, %res1 + ret <8 x double> %res2 +} + +declare <16 x float> @llvm.x86.avx512.mask.range.ps.512(<16 x float>, <16 x float>, i32, <16 x float>, i16, i32) + +; CHECK-LABEL: @test_int_x86_avx512_mask_range_ps_512 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vrangeps +; CKECK: {%k1} +; CHECK: vrangeps +; CHECK: {sae} +define <16 x float>@test_int_x86_avx512_mask_range_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x3, i16 %x4) { + %res = call <16 x float> @llvm.x86.avx512.mask.range.ps.512(<16 x float> %x0, <16 x float> %x1, i32 88, <16 x float> %x3, i16 %x4, i32 4) + %res1 = call <16 x float> @llvm.x86.avx512.mask.range.ps.512(<16 x float> %x0, <16 x float> %x1, i32 4, <16 x float> %x3, i16 -1, i32 8) + %res2 = fadd <16 x float> %res, %res1 + ret <16 x float> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.reduce.ss(<4 x float>, <4 x float>,<4 x float>, i8, i32, i32) + +; CHECK-LABEL: @test_int_x86_avx512_mask_reduce_ss +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vreducess +; CKECK: {%k1} +; CHECK: vreducess +; CHECK: {sae} +define <4 x float>@test_int_x86_avx512_mask_reduce_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 %x4) { + %res = call <4 x float> @llvm.x86.avx512.mask.reduce.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 %x4, i32 4, i32 4) + %res1 = call <4 x float> @llvm.x86.avx512.mask.reduce.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 -1, i32 4, i32 8) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.range.ss(<4 x float>, <4 x float>,<4 x float>, i8, i32, i32) +; CHECK-LABEL: @test_int_x86_avx512_mask_range_ss +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vrangess +; CHECK: {sae} +; CKECK: {%k1} +; CHECK: vrangess +; CHECK: {sae} +define <4 x float>@test_int_x86_avx512_mask_range_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 %x4) { + %res = call <4 x float> @llvm.x86.avx512.mask.range.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 %x4, i32 4, i32 8) + %res1 = call <4 x float> @llvm.x86.avx512.mask.range.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 -1, i32 4, i32 8) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <2 x double> @llvm.x86.avx512.mask.reduce.sd(<2 x double>, <2 x double>,<2 x double>, i8, i32, i32) + +; CHECK-LABEL: @test_int_x86_avx512_mask_reduce_sd +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vreducesd +; CKECK: {%k1} +; CHECK: vreducesd +; CHECK: {sae} +define <2 x double>@test_int_x86_avx512_mask_reduce_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 %x4) { + %res = call <2 x double> @llvm.x86.avx512.mask.reduce.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 %x4, i32 4, i32 4) + %res1 = call <2 x double> @llvm.x86.avx512.mask.reduce.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 -1, i32 4, i32 8) + %res2 = fadd <2 x double> %res, %res1 + ret <2 x double> %res2 +} + +declare <2 x double> @llvm.x86.avx512.mask.range.sd(<2 x double>, <2 x double>,<2 x double>, i8, i32, i32) +; CHECK-LABEL: @test_int_x86_avx512_mask_range_sd +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vrangesd +; CKECK: {%k1} +; CHECK: vrangesd +; CHECK: {sae} +define <2 x double>@test_int_x86_avx512_mask_range_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 %x4) { + %res = call <2 x double> @llvm.x86.avx512.mask.range.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 %x4, i32 4, i32 4) + %res1 = call <2 x double> @llvm.x86.avx512.mask.range.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 -1, i32 4, i32 8) + %res2 = fadd <2 x double> %res, %res1 + ret <2 x double> %res2 +} + + +declare <2 x double> @llvm.x86.avx512.mask.vextractf64x2.512(<8 x double>, i32, <2 x double>, i8) + +define <2 x double>@test_int_x86_avx512_mask_vextractf64x2_512(<8 x double> %x0, <2 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vextractf64x2_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vextractf64x2 $1, %zmm0, %xmm1 {%k1} +; CHECK-NEXT: vextractf64x2 $1, %zmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vextractf64x2 $1, %zmm0, %xmm0 +; CHECK-NEXT: vaddpd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: vaddpd %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.vextractf64x2.512(<8 x double> %x0,i32 1, <2 x double> %x2, i8 %x3) + %res2 = call <2 x double> @llvm.x86.avx512.mask.vextractf64x2.512(<8 x double> %x0,i32 1, <2 x double> zeroinitializer, i8 %x3) + %res1 = call <2 x double> @llvm.x86.avx512.mask.vextractf64x2.512(<8 x double> %x0,i32 1, <2 x double> zeroinitializer, i8 -1) + %res3 = fadd <2 x double> %res, %res1 + %res4 = fadd <2 x double> %res2, %res3 + ret <2 x double> %res4 +} + +declare <8 x float> @llvm.x86.avx512.mask.vextractf32x8.512(<16 x float>, i32, <8 x float>, i8) + +define <8 x float>@test_int_x86_avx512_mask_vextractf32x8(<16 x float> %x0, <8 x float> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vextractf32x8: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vextractf32x8 $1, %zmm0, %ymm1 {%k1} +; CHECK-NEXT: vextractf32x8 $1, %zmm0, %ymm2 {%k1} {z} +; CHECK-NEXT: vextractf32x8 $1, %zmm0, %ymm0 +; CHECK-NEXT: vaddps %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: vaddps %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.avx512.mask.vextractf32x8.512(<16 x float> %x0,i32 1, <8 x float> %x2, i8 %x3) + %res2 = call <8 x float> @llvm.x86.avx512.mask.vextractf32x8.512(<16 x float> %x0,i32 1, <8 x float> zeroinitializer, i8 %x3) + %res1 = call <8 x float> @llvm.x86.avx512.mask.vextractf32x8.512(<16 x float> %x0,i32 1, <8 x float> zeroinitializer, i8 -1) + %res3 = fadd <8 x float> %res, %res1 + %res4 = fadd <8 x float> %res2, %res3 + ret <8 x float> %res4 +} + +declare <16 x float> @llvm.x86.avx512.mask.insertf32x8.512(<16 x float>, <8 x float>, i32, <16 x float>, i16) + +define <16 x float>@test_int_x86_avx512_mask_insertf32x8_512(<16 x float> %x0, <8 x float> %x1, <16 x float> %x3, i16 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_insertf32x8_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vinsertf32x8 $1, %ymm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vinsertf32x8 $1, %ymm1, %zmm0, %zmm3 {%k1} {z} +; CHECK-NEXT: vinsertf32x8 $1, %ymm1, %zmm0, %zmm0 +; CHECK-NEXT: vaddps %zmm3, %zmm2, %zmm1 +; CHECK-NEXT: vaddps %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.insertf32x8.512(<16 x float> %x0, <8 x float> %x1, i32 1, <16 x float> %x3, i16 %x4) + %res1 = call <16 x float> @llvm.x86.avx512.mask.insertf32x8.512(<16 x float> %x0, <8 x float> %x1, i32 1, <16 x float> zeroinitializer, i16 %x4) + %res2 = call <16 x float> @llvm.x86.avx512.mask.insertf32x8.512(<16 x float> %x0, <8 x float> %x1, i32 1, <16 x float> %x3, i16 -1) + %res3 = fadd <16 x float> %res, %res1 + %res4 = fadd <16 x float> %res2, %res3 + ret <16 x float> %res4 +} + +declare <8 x double> @llvm.x86.avx512.mask.insertf64x2.512(<8 x double>, <2 x double>, i32, <8 x double>, i8) + +define <8 x double>@test_int_x86_avx512_mask_insertf64x2_512(<8 x double> %x0, <2 x double> %x1,<8 x double> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_insertf64x2_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vinsertf64x2 $1, %xmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vinsertf64x2 $1, %xmm1, %zmm0, %zmm3 {%k1} {z} +; CHECK-NEXT: vinsertf64x2 $1, %xmm1, %zmm0, %zmm0 +; CHECK-NEXT: vaddpd %zmm3, %zmm2, %zmm1 +; CHECK-NEXT: vaddpd %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x double> @llvm.x86.avx512.mask.insertf64x2.512(<8 x double> %x0, <2 x double> %x1, i32 1, <8 x double> %x3, i8 %x4) + %res1 = call <8 x double> @llvm.x86.avx512.mask.insertf64x2.512(<8 x double> %x0, <2 x double> %x1, i32 1, <8 x double> zeroinitializer, i8 %x4) + %res2 = call <8 x double> @llvm.x86.avx512.mask.insertf64x2.512(<8 x double> %x0, <2 x double> %x1, i32 1, <8 x double> %x3, i8 -1) + %res3 = fadd <8 x double> %res, %res1 + %res4 = fadd <8 x double> %res3, %res2 + ret <8 x double> %res4 +} + +declare <16 x i32> @llvm.x86.avx512.mask.inserti32x8.512(<16 x i32>, <8 x i32>, i32, <16 x i32>, i16) + +define <16 x i32>@test_int_x86_avx512_mask_inserti32x8_512(<16 x i32> %x0, <8 x i32> %x1, <16 x i32> %x3, i16 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_inserti32x8_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vinserti32x8 $1, %ymm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vinserti32x8 $1, %ymm1, %zmm0, %zmm3 {%k1} {z} +; CHECK-NEXT: vinserti32x8 $1, %ymm1, %zmm0, %zmm0 +; CHECK-NEXT: vpaddd %zmm3, %zmm2, %zmm1 +; CHECK-NEXT: vpaddd %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x i32> @llvm.x86.avx512.mask.inserti32x8.512(<16 x i32> %x0, <8 x i32> %x1, i32 1, <16 x i32> %x3, i16 %x4) + %res1 = call <16 x i32> @llvm.x86.avx512.mask.inserti32x8.512(<16 x i32> %x0, <8 x i32> %x1, i32 1, <16 x i32> zeroinitializer, i16 %x4) + %res2 = call <16 x i32> @llvm.x86.avx512.mask.inserti32x8.512(<16 x i32> %x0, <8 x i32> %x1, i32 1, <16 x i32> %x3, i16 -1) + %res3 = add <16 x i32> %res, %res1 + %res4 = add <16 x i32> %res3, %res2 + ret <16 x i32> %res4 +} + +declare <8 x i64> @llvm.x86.avx512.mask.inserti64x2.512(<8 x i64>, <2 x i64>, i32, <8 x i64>, i8) + +define <8 x i64>@test_int_x86_avx512_mask_inserti64x2_512(<8 x i64> %x0, <2 x i64> %x1, <8 x i64> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_inserti64x2_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vinserti64x2 $1, %xmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vinserti64x2 $1, %xmm1, %zmm0, %zmm3 {%k1} {z} +; CHECK-NEXT: vinserti64x2 $1, %xmm1, %zmm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm3, %zmm2, %zmm1 +; CHECK-NEXT: vpaddq %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.mask.inserti64x2.512(<8 x i64> %x0, <2 x i64> %x1, i32 1, <8 x i64> %x3, i8 %x4) + %res1 = call <8 x i64> @llvm.x86.avx512.mask.inserti64x2.512(<8 x i64> %x0, <2 x i64> %x1, i32 1, <8 x i64> zeroinitializer, i8 %x4) + %res2 = call <8 x i64> @llvm.x86.avx512.mask.inserti64x2.512(<8 x i64> %x0, <2 x i64> %x1, i32 1, <8 x i64> %x3, i8 -1) + %res3 = add <8 x i64> %res, %res1 + %res4 = add <8 x i64> %res2, %res3 + ret <8 x i64> %res4 +} + +declare i8 @llvm.x86.avx512.mask.fpclass.pd.512(<8 x double>, i32, i8) + +; CHECK-LABEL: @test_int_x86_avx512_mask_fpclass_pd_512 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vfpclasspd +; CHECK: {%k1} +; CHECK: vfpclasspd +; CHECK: kmovb %k0 +define i8 @test_int_x86_avx512_mask_fpclass_pd_512(<8 x double> %x0, i8 %x1) { + %res = call i8 @llvm.x86.avx512.mask.fpclass.pd.512(<8 x double> %x0, i32 2, i8 %x1) + %res1 = call i8 @llvm.x86.avx512.mask.fpclass.pd.512(<8 x double> %x0, i32 4, i8 -1) + %res2 = add i8 %res, %res1 + ret i8 %res2 +} +declare i16 @llvm.x86.avx512.mask.fpclass.ps.512(<16 x float>, i32, i16) + +; CHECK-LABEL: @test_int_x86_avx512_mask_fpclass_ps_512 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vfpclassps +; CHECK: vfpclassps +; CHECK: {%k1} +; CHECK: kmov +define i16@test_int_x86_avx512_mask_fpclass_ps_512(<16 x float> %x0, i16 %x1) { + %res = call i16 @llvm.x86.avx512.mask.fpclass.ps.512(<16 x float> %x0, i32 4, i16 %x1) + %res1 = call i16 @llvm.x86.avx512.mask.fpclass.ps.512(<16 x float> %x0, i32 4, i16 -1) + %res2 = add i16 %res, %res1 + ret i16 %res2 +} + +declare i8 @llvm.x86.avx512.mask.fpclass.sd(<2 x double>, i32, i8) + +; CHECK-LABEL: @test_int_x86_avx512_mask_fpclass_sd +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vfpclasssd +; CHECK: %k0 {%k1} +; CHECK: vfpclasssd +; CHECK: %k0 +define i8 @test_int_x86_avx512_mask_fpclass_sd(<2 x double> %x0, i8 %x1) { + %res = call i8 @llvm.x86.avx512.mask.fpclass.sd(<2 x double> %x0, i32 2, i8 %x1) + %res1 = call i8 @llvm.x86.avx512.mask.fpclass.sd(<2 x double> %x0, i32 4, i8 -1) + %res2 = add i8 %res, %res1 + ret i8 %res2 +} + +declare i8 @llvm.x86.avx512.mask.fpclass.ss(<4 x float>, i32, i8) + +; CHECK-LABEL: @test_int_x86_avx512_mask_fpclass_ss +; CHECK-NOT: call +; CHECK: kmovw +; CHECK: vfpclassss +; CHECK: %k0 +; CHECK: {%k1} +; CHECK: kmovw +; CHECK: vfpclassss +; CHECK: %k0 +define i8 @test_int_x86_avx512_mask_fpclass_ss(<4 x float> %x0, i8 %x1) { + %res = call i8 @llvm.x86.avx512.mask.fpclass.ss(<4 x float> %x0, i32 4, i8 %x1) + %res1 = call i8 @llvm.x86.avx512.mask.fpclass.ss(<4 x float> %x0, i32 4, i8 -1) + %res2 = add i8 %res, %res1 + ret i8 %res2 +} + +declare <16 x float> @llvm.x86.avx512.mask.broadcastf32x2.512(<4 x float>, <16 x float>, i16) + +define <16 x float>@test_int_x86_avx512_mask_broadcastf32x2_512(<4 x float> %x0, <16 x float> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_broadcastf32x2_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vbroadcastf32x2 %xmm0, %zmm1 {%k1} +; CHECK-NEXT: vbroadcastf32x2 %xmm0, %zmm2 {%k1} {z} +; CHECK-NEXT: vbroadcastf32x2 %xmm0, %zmm0 +; CHECK-NEXT: vaddps %zmm2, %zmm1, %zmm1 +; CHECK-NEXT: vaddps %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x float> @llvm.x86.avx512.mask.broadcastf32x2.512(<4 x float> %x0, <16 x float> %x2, i16 %x3) + %res1 = call <16 x float> @llvm.x86.avx512.mask.broadcastf32x2.512(<4 x float> %x0, <16 x float> zeroinitializer, i16 %x3) + %res2 = call <16 x float> @llvm.x86.avx512.mask.broadcastf32x2.512(<4 x float> %x0, <16 x float> %x2, i16 -1) + %res3 = fadd <16 x float> %res, %res1 + %res4 = fadd <16 x float> %res3, %res2 + ret <16 x float> %res4 +} + +declare <16 x i32> @llvm.x86.avx512.mask.broadcasti32x2.512(<4 x i32>, <16 x i32>, i16) + +define <16 x i32>@test_int_x86_avx512_mask_broadcasti32x2_512(<4 x i32> %x0, <16 x i32> %x2, i16 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_broadcasti32x2_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vbroadcasti32x2 %xmm0, %zmm1 {%k1} +; CHECK-NEXT: vbroadcasti32x2 %xmm0, %zmm2 {%k1} {z} +; CHECK-NEXT: vbroadcasti32x2 %xmm0, %zmm0 +; CHECK-NEXT: vpaddd %zmm2, %zmm1, %zmm1 +; CHECK-NEXT: vpaddd %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x i32> @llvm.x86.avx512.mask.broadcasti32x2.512(<4 x i32> %x0, <16 x i32> %x2, i16 %x3) + %res1 = call <16 x i32> @llvm.x86.avx512.mask.broadcasti32x2.512(<4 x i32> %x0, <16 x i32> zeroinitializer, i16 %x3) + %res2 = call <16 x i32> @llvm.x86.avx512.mask.broadcasti32x2.512(<4 x i32> %x0, <16 x i32> %x2, i16 -1) + %res3 = add <16 x i32> %res, %res1 + %res4 = add <16 x i32> %res3, %res2 + ret <16 x i32> %res4 +} + +declare i16 @llvm.x86.avx512.cvtd2mask.512(<16 x i32>) + +define i16@test_int_x86_avx512_cvtd2mask_512(<16 x i32> %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtd2mask_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovd2m %zmm0, %k0 +; CHECK-NEXT: kmovw %k0, %eax +; CHECK-NEXT: retq + %res = call i16 @llvm.x86.avx512.cvtd2mask.512(<16 x i32> %x0) + ret i16 %res +} + +declare i8 @llvm.x86.avx512.cvtq2mask.512(<8 x i64>) + +define i8@test_int_x86_avx512_cvtq2mask_512(<8 x i64> %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtq2mask_512: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovq2m %zmm0, %k0 +; CHECK-NEXT: kmovb %k0, %eax +; CHECK-NEXT: retq + %res = call i8 @llvm.x86.avx512.cvtq2mask.512(<8 x i64> %x0) + ret i8 %res +} + +declare <16 x i32> @llvm.x86.avx512.cvtmask2d.512(i16) + +define <16 x i32>@test_int_x86_avx512_cvtmask2d_512(i16 %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtmask2d_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k0 +; CHECK-NEXT: vpmovm2d %k0, %zmm0 +; CHECK-NEXT: retq + %res = call <16 x i32> @llvm.x86.avx512.cvtmask2d.512(i16 %x0) + ret <16 x i32> %res +} + +declare <8 x i64> @llvm.x86.avx512.cvtmask2q.512(i8) + +define <8 x i64>@test_int_x86_avx512_cvtmask2q_512(i8 %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtmask2q_512: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k0 +; CHECK-NEXT: vpmovm2q %k0, %zmm0 +; CHECK-NEXT: retq + %res = call <8 x i64> @llvm.x86.avx512.cvtmask2q.512(i8 %x0) + ret <8 x i64> %res +} + +declare <16 x float> @llvm.x86.avx512.mask.broadcastf32x8.512(<8 x float>, <16 x float>, i16) + +define <16 x float>@test_int_x86_avx512_mask_broadcastf32x8_512(<8 x float> %x0, <16 x float> %x2, i16 %mask) { +; CHECK-LABEL: test_int_x86_avx512_mask_broadcastf32x8_512: +; CHECK: kmovw %edi, %k1 +; CHECK: vshuff32x4 $68, %zmm0, %zmm0, %zmm2 {%k1} {z} +; CHECK: vshuff32x4 $68, %zmm0, %zmm0, %zmm1 {%k1} +; CHECK: vshuff32x4 $68, %zmm0, %zmm0, %zmm0 +; CHECK: vaddps %zmm1, %zmm0, %zmm0 +; CHECK: vaddps %zmm0, %zmm2, %zmm0 + + %res1 = call <16 x float> @llvm.x86.avx512.mask.broadcastf32x8.512(<8 x float> %x0, <16 x float> %x2, i16 -1) + %res2 = call <16 x float> @llvm.x86.avx512.mask.broadcastf32x8.512(<8 x float> %x0, <16 x float> %x2, i16 %mask) + %res3 = call <16 x float> @llvm.x86.avx512.mask.broadcastf32x8.512(<8 x float> %x0, <16 x float> zeroinitializer, i16 %mask) + %res4 = fadd <16 x float> %res1, %res2 + %res5 = fadd <16 x float> %res3, %res4 + ret <16 x float> %res5 +} + +declare <8 x double> @llvm.x86.avx512.mask.broadcastf64x2.512(<2 x double>, <8 x double>, i8) + +define <8 x double>@test_int_x86_avx512_mask_broadcastf64x2_512(<2 x double> %x0, <8 x double> %x2, i8 %mask) { +; CHECK-LABEL: test_int_x86_avx512_mask_broadcastf64x2_512: +; CHECK: kmovb %edi, %k1 +; CHECK: vshuff64x2 $0, %zmm0, %zmm0, %zmm2 {%k1} {z} +; CHECK: vshuff64x2 $0, %zmm0, %zmm0, %zmm1 {%k1} +; CHECK: vshuff64x2 $0, %zmm0, %zmm0, %zmm0 +; CHECK: vaddpd %zmm1, %zmm0, %zmm0 +; CHECK: vaddpd %zmm0, %zmm2, %zmm0 + + %res1 = call <8 x double> @llvm.x86.avx512.mask.broadcastf64x2.512(<2 x double> %x0, <8 x double> %x2, i8 -1) + %res2 = call <8 x double> @llvm.x86.avx512.mask.broadcastf64x2.512(<2 x double> %x0, <8 x double> %x2, i8 %mask) + %res3 = call <8 x double> @llvm.x86.avx512.mask.broadcastf64x2.512(<2 x double> %x0, <8 x double> zeroinitializer, i8 %mask) + %res4 = fadd <8 x double> %res1, %res2 + %res5 = fadd <8 x double> %res3, %res4 + ret <8 x double> %res5 +} + +declare <16 x i32> @llvm.x86.avx512.mask.broadcasti32x8.512(<8 x i32>, <16 x i32>, i16) + +define <16 x i32>@test_int_x86_avx512_mask_broadcasti32x8_512(<8 x i32> %x0, <16 x i32> %x2, i16 %mask) { +; CHECK-LABEL: test_int_x86_avx512_mask_broadcasti32x8_512: +; CHECK: kmovw %edi, %k1 +; CHECK: vshufi32x4 $68, %zmm0, %zmm0, %zmm2 {%k1} {z} +; CHECK: vshufi32x4 $68, %zmm0, %zmm0, %zmm1 {%k1} +; CHECK: vshufi32x4 $68, %zmm0, %zmm0, %zmm0 +; CHECK: vpaddd %zmm1, %zmm0, %zmm0 +; CHECK: vpaddd %zmm0, %zmm2, %zmm0 + + %res1 = call <16 x i32> @llvm.x86.avx512.mask.broadcasti32x8.512(<8 x i32> %x0, <16 x i32> %x2, i16 -1) + %res2 = call <16 x i32> @llvm.x86.avx512.mask.broadcasti32x8.512(<8 x i32> %x0, <16 x i32> %x2, i16 %mask) + %res3 = call <16 x i32> @llvm.x86.avx512.mask.broadcasti32x8.512(<8 x i32> %x0, <16 x i32> zeroinitializer, i16 %mask) + %res4 = add <16 x i32> %res1, %res2 + %res5 = add <16 x i32> %res3, %res4 + ret <16 x i32> %res5 +} + +declare <8 x i64> @llvm.x86.avx512.mask.broadcasti64x2.512(<2 x i64>, <8 x i64>, i8) + +define <8 x i64>@test_int_x86_avx512_mask_broadcasti64x2_512(<2 x i64> %x0, <8 x i64> %x2, i8 %mask) { +; CHECK-LABEL: test_int_x86_avx512_mask_broadcasti64x2_512: +; CHECK: kmovb %edi, %k1 +; CHECK: vshufi64x2 $0, %zmm0, %zmm0, %zmm2 {%k1} {z} +; CHECK: vshufi64x2 $0, %zmm0, %zmm0, %zmm1 {%k1} +; CHECK: vshufi64x2 $0, %zmm0, %zmm0, %zmm0 +; CHECK: vpaddq %zmm1, %zmm0, %zmm0 +; CHECK: vpaddq %zmm0, %zmm2, %zmm0 + + %res1 = call <8 x i64> @llvm.x86.avx512.mask.broadcasti64x2.512(<2 x i64> %x0, <8 x i64> %x2, i8 -1) + %res2 = call <8 x i64> @llvm.x86.avx512.mask.broadcasti64x2.512(<2 x i64> %x0, <8 x i64> %x2, i8 %mask) + %res3 = call <8 x i64> @llvm.x86.avx512.mask.broadcasti64x2.512(<2 x i64> %x0, <8 x i64> zeroinitializer, i8 %mask) + %res4 = add <8 x i64> %res1, %res2 + %res5 = add <8 x i64> %res3, %res4 + ret <8 x i64> %res5 +} diff --git a/test/CodeGen/X86/avx512dqvl-intrinsics.ll b/test/CodeGen/X86/avx512dqvl-intrinsics.ll index c577abee6640..2065322009da 100644 --- a/test/CodeGen/X86/avx512dqvl-intrinsics.ll +++ b/test/CodeGen/X86/avx512dqvl-intrinsics.ll @@ -1134,7 +1134,7 @@ define <16 x float> @test_mask_xor_ps_rmb_512(<16 x float> %a, float* %ptr_b) { define <16 x float> @test_mask_xor_ps_rmbk_512(<16 x float> %a, float* %ptr_b, <16 x float> %passThru, i16 %mask) { ;CHECK-LABEL: test_mask_xor_ps_rmbk_512 - ;CHECK: vxorps (%rdi){1to16}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x59,0x57,0x0f] + ;CHECK: vxorps (%rdi){1to16}, %zmm0, %zmm1 {%k1} %q = load float, float* %ptr_b %vecinit.i = insertelement <16 x float> undef, float %q, i32 0 %b = shufflevector <16 x float> %vecinit.i, <16 x float> undef, <16 x i32> zeroinitializer @@ -1144,7 +1144,7 @@ define <16 x float> @test_mask_xor_ps_rmbk_512(<16 x float> %a, float* %ptr_b, < define <16 x float> @test_mask_xor_ps_rmbkz_512(<16 x float> %a, float* %ptr_b, i16 %mask) { ;CHECK-LABEL: test_mask_xor_ps_rmbkz_512 - ;CHECK: vxorps (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xd9,0x57,0x07] + ;CHECK: vxorps (%rdi){1to16}, %zmm0, %zmm0 {%k1} {z} %q = load float, float* %ptr_b %vecinit.i = insertelement <16 x float> undef, float %q, i32 0 %b = shufflevector <16 x float> %vecinit.i, <16 x float> undef, <16 x i32> zeroinitializer @@ -1152,4 +1152,816 @@ define <16 x float> @test_mask_xor_ps_rmbkz_512(<16 x float> %a, float* %ptr_b, ret <16 x float> %res } -declare <16 x float> @llvm.x86.avx512.mask.xor.ps.512(<16 x float>, <16 x float>, <16 x float>, i16) \ No newline at end of file +declare <16 x float> @llvm.x86.avx512.mask.xor.ps.512(<16 x float>, <16 x float>, <16 x float>, i16) + +declare <2 x i64> @llvm.x86.avx512.mask.cvtpd2qq.128(<2 x double>, <2 x i64>, i8) + +define <2 x i64>@test_int_x86_avx512_mask_cvt_pd2qq_128(<2 x double> %x0, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2qq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtpd2qq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtpd2qq %xmm0, %xmm0 +; CHECK-NEXT: vpaddq %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x i64> @llvm.x86.avx512.mask.cvtpd2qq.128(<2 x double> %x0, <2 x i64> %x1, i8 %x2) + %res1 = call <2 x i64> @llvm.x86.avx512.mask.cvtpd2qq.128(<2 x double> %x0, <2 x i64> %x1, i8 -1) + %res2 = add <2 x i64> %res, %res1 + ret <2 x i64> %res2 +} + +declare <4 x i64> @llvm.x86.avx512.mask.cvtpd2qq.256(<4 x double>, <4 x i64>, i8) + +define <4 x i64>@test_int_x86_avx512_mask_cvt_pd2qq_256(<4 x double> %x0, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2qq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtpd2qq %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtpd2qq %ymm0, %ymm0 +; CHECK-NEXT: vpaddq %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.mask.cvtpd2qq.256(<4 x double> %x0, <4 x i64> %x1, i8 %x2) + %res1 = call <4 x i64> @llvm.x86.avx512.mask.cvtpd2qq.256(<4 x double> %x0, <4 x i64> %x1, i8 -1) + %res2 = add <4 x i64> %res, %res1 + ret <4 x i64> %res2 +} + +declare <2 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.128(<2 x double>, <2 x i64>, i8) + +define <2 x i64>@test_int_x86_avx512_mask_cvt_pd2uqq_128(<2 x double> %x0, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2uqq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtpd2uqq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtpd2uqq %xmm0, %xmm0 +; CHECK-NEXT: vpaddq %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.128(<2 x double> %x0, <2 x i64> %x1, i8 %x2) + %res1 = call <2 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.128(<2 x double> %x0, <2 x i64> %x1, i8 -1) + %res2 = add <2 x i64> %res, %res1 + ret <2 x i64> %res2 +} + +declare <4 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.256(<4 x double>, <4 x i64>, i8) + +define <4 x i64>@test_int_x86_avx512_mask_cvt_pd2uqq_256(<4 x double> %x0, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2uqq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtpd2uqq %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtpd2uqq %ymm0, %ymm0 +; CHECK-NEXT: vpaddq %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.256(<4 x double> %x0, <4 x i64> %x1, i8 %x2) + %res1 = call <4 x i64> @llvm.x86.avx512.mask.cvtpd2uqq.256(<4 x double> %x0, <4 x i64> %x1, i8 -1) + %res2 = add <4 x i64> %res, %res1 + ret <4 x i64> %res2 +} + +declare <2 x i64> @llvm.x86.avx512.mask.cvtps2qq.128(<4 x float>, <2 x i64>, i8) + +define <2 x i64>@test_int_x86_avx512_mask_cvt_ps2qq_128(<4 x float> %x0, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2qq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtps2qq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtps2qq %xmm0, %xmm0 +; CHECK-NEXT: vpaddq %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x i64> @llvm.x86.avx512.mask.cvtps2qq.128(<4 x float> %x0, <2 x i64> %x1, i8 %x2) + %res1 = call <2 x i64> @llvm.x86.avx512.mask.cvtps2qq.128(<4 x float> %x0, <2 x i64> %x1, i8 -1) + %res2 = add <2 x i64> %res, %res1 + ret <2 x i64> %res2 +} + +declare <4 x i64> @llvm.x86.avx512.mask.cvtps2qq.256(<4 x float>, <4 x i64>, i8) + +define <4 x i64>@test_int_x86_avx512_mask_cvt_ps2qq_256(<4 x float> %x0, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2qq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtps2qq %xmm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtps2qq %xmm0, %ymm0 +; CHECK-NEXT: vpaddq %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.mask.cvtps2qq.256(<4 x float> %x0, <4 x i64> %x1, i8 %x2) + %res1 = call <4 x i64> @llvm.x86.avx512.mask.cvtps2qq.256(<4 x float> %x0, <4 x i64> %x1, i8 -1) + %res2 = add <4 x i64> %res, %res1 + ret <4 x i64> %res2 +} + +declare <2 x i64> @llvm.x86.avx512.mask.cvtps2uqq.128(<4 x float>, <2 x i64>, i8) + +define <2 x i64>@test_int_x86_avx512_mask_cvt_ps2uqq_128(<4 x float> %x0, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2uqq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtps2uqq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtps2uqq %xmm0, %xmm0 +; CHECK-NEXT: vpaddq %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x i64> @llvm.x86.avx512.mask.cvtps2uqq.128(<4 x float> %x0, <2 x i64> %x1, i8 %x2) + %res1 = call <2 x i64> @llvm.x86.avx512.mask.cvtps2uqq.128(<4 x float> %x0, <2 x i64> %x1, i8 -1) + %res2 = add <2 x i64> %res, %res1 + ret <2 x i64> %res2 +} + +declare <4 x i64> @llvm.x86.avx512.mask.cvtps2uqq.256(<4 x float>, <4 x i64>, i8) + +define <4 x i64>@test_int_x86_avx512_mask_cvt_ps2uqq_256(<4 x float> %x0, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2uqq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtps2uqq %xmm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtps2uqq %xmm0, %ymm0 +; CHECK-NEXT: vpaddq %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.mask.cvtps2uqq.256(<4 x float> %x0, <4 x i64> %x1, i8 %x2) + %res1 = call <4 x i64> @llvm.x86.avx512.mask.cvtps2uqq.256(<4 x float> %x0, <4 x i64> %x1, i8 -1) + %res2 = add <4 x i64> %res, %res1 + ret <4 x i64> %res2 +} + +declare <2 x double> @llvm.x86.avx512.mask.cvtqq2pd.128(<2 x i64>, <2 x double>, i8) + +define <2 x double>@test_int_x86_avx512_mask_cvt_qq2pd_128(<2 x i64> %x0, <2 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_qq2pd_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtqq2pd %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtqq2pd %xmm0, %xmm0 +; CHECK-NEXT: vaddpd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.cvtqq2pd.128(<2 x i64> %x0, <2 x double> %x1, i8 %x2) + %res1 = call <2 x double> @llvm.x86.avx512.mask.cvtqq2pd.128(<2 x i64> %x0, <2 x double> %x1, i8 -1) + %res2 = fadd <2 x double> %res, %res1 + ret <2 x double> %res2 +} + +declare <4 x double> @llvm.x86.avx512.mask.cvtqq2pd.256(<4 x i64>, <4 x double>, i8) + +define <4 x double>@test_int_x86_avx512_mask_cvt_qq2pd_256(<4 x i64> %x0, <4 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_qq2pd_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtqq2pd %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtqq2pd %ymm0, %ymm0 +; CHECK-NEXT: vaddpd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.avx512.mask.cvtqq2pd.256(<4 x i64> %x0, <4 x double> %x1, i8 %x2) + %res1 = call <4 x double> @llvm.x86.avx512.mask.cvtqq2pd.256(<4 x i64> %x0, <4 x double> %x1, i8 -1) + %res2 = fadd <4 x double> %res, %res1 + ret <4 x double> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.128(<2 x i64>, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_cvt_qq2ps_128(<2 x i64> %x0, <4 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_qq2ps_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtqq2ps %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtqq2ps %xmm0, %xmm0 +; CHECK-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.128(<2 x i64> %x0, <4 x float> %x1, i8 %x2) + %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.128(<2 x i64> %x0, <4 x float> %x1, i8 -1) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.256(<4 x i64>, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_cvt_qq2ps_256(<4 x i64> %x0, <4 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_qq2ps_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtqq2ps %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtqq2ps %ymm0, %xmm0 +; CHECK-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.256(<4 x i64> %x0, <4 x float> %x1, i8 %x2) + %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.256(<4 x i64> %x0, <4 x float> %x1, i8 -1) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <2 x i64> @llvm.x86.avx512.mask.cvttpd2qq.128(<2 x double>, <2 x i64>, i8) + +define <2 x i64>@test_int_x86_avx512_mask_cvtt_pd2qq_128(<2 x double> %x0, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2qq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvttpd2qq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvttpd2qq %xmm0, %xmm0 +; CHECK-NEXT: vpaddq %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x i64> @llvm.x86.avx512.mask.cvttpd2qq.128(<2 x double> %x0, <2 x i64> %x1, i8 %x2) + %res1 = call <2 x i64> @llvm.x86.avx512.mask.cvttpd2qq.128(<2 x double> %x0, <2 x i64> %x1, i8 -1) + %res2 = add <2 x i64> %res, %res1 + ret <2 x i64> %res2 +} + +declare <4 x i64> @llvm.x86.avx512.mask.cvttpd2qq.256(<4 x double>, <4 x i64>, i8) + +define <4 x i64>@test_int_x86_avx512_mask_cvtt_pd2qq_256(<4 x double> %x0, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2qq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvttpd2qq %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vcvttpd2qq %ymm0, %ymm0 +; CHECK-NEXT: vpaddq %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.mask.cvttpd2qq.256(<4 x double> %x0, <4 x i64> %x1, i8 %x2) + %res1 = call <4 x i64> @llvm.x86.avx512.mask.cvttpd2qq.256(<4 x double> %x0, <4 x i64> %x1, i8 -1) + %res2 = add <4 x i64> %res, %res1 + ret <4 x i64> %res2 +} + +declare <2 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.128(<2 x double>, <2 x i64>, i8) + +define <2 x i64>@test_int_x86_avx512_mask_cvtt_pd2uqq_128(<2 x double> %x0, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2uqq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvttpd2uqq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvttpd2uqq %xmm0, %xmm0 +; CHECK-NEXT: vpaddq %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.128(<2 x double> %x0, <2 x i64> %x1, i8 %x2) + %res1 = call <2 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.128(<2 x double> %x0, <2 x i64> %x1, i8 -1) + %res2 = add <2 x i64> %res, %res1 + ret <2 x i64> %res2 +} + +declare <4 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.256(<4 x double>, <4 x i64>, i8) + +define <4 x i64>@test_int_x86_avx512_mask_cvtt_pd2uqq_256(<4 x double> %x0, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2uqq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvttpd2uqq %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vcvttpd2uqq %ymm0, %ymm0 +; CHECK-NEXT: vpaddq %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.256(<4 x double> %x0, <4 x i64> %x1, i8 %x2) + %res1 = call <4 x i64> @llvm.x86.avx512.mask.cvttpd2uqq.256(<4 x double> %x0, <4 x i64> %x1, i8 -1) + %res2 = add <4 x i64> %res, %res1 + ret <4 x i64> %res2 +} + +declare <2 x i64> @llvm.x86.avx512.mask.cvttps2qq.128(<4 x float>, <2 x i64>, i8) + +define <2 x i64>@test_int_x86_avx512_mask_cvtt_ps2qq_128(<4 x float> %x0, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2qq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvttps2qq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvttps2qq %xmm0, %xmm0 +; CHECK-NEXT: vpaddq %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x i64> @llvm.x86.avx512.mask.cvttps2qq.128(<4 x float> %x0, <2 x i64> %x1, i8 %x2) + %res1 = call <2 x i64> @llvm.x86.avx512.mask.cvttps2qq.128(<4 x float> %x0, <2 x i64> %x1, i8 -1) + %res2 = add <2 x i64> %res, %res1 + ret <2 x i64> %res2 +} + +declare <4 x i64> @llvm.x86.avx512.mask.cvttps2qq.256(<4 x float>, <4 x i64>, i8) + +define <4 x i64>@test_int_x86_avx512_mask_cvtt_ps2qq_256(<4 x float> %x0, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2qq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvttps2qq %xmm0, %ymm1 {%k1} +; CHECK-NEXT: vcvttps2qq %xmm0, %ymm0 +; CHECK-NEXT: vpaddq %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.mask.cvttps2qq.256(<4 x float> %x0, <4 x i64> %x1, i8 %x2) + %res1 = call <4 x i64> @llvm.x86.avx512.mask.cvttps2qq.256(<4 x float> %x0, <4 x i64> %x1, i8 -1) + %res2 = add <4 x i64> %res, %res1 + ret <4 x i64> %res2 +} + +declare <2 x double> @llvm.x86.avx512.mask.cvtuqq2pd.128(<2 x i64>, <2 x double>, i8) + +define <2 x double>@test_int_x86_avx512_mask_cvt_uqq2pd_128(<2 x i64> %x0, <2 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_uqq2pd_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtuqq2pd %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtuqq2pd %xmm0, %xmm0 +; CHECK-NEXT: vaddpd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.cvtuqq2pd.128(<2 x i64> %x0, <2 x double> %x1, i8 %x2) + %res1 = call <2 x double> @llvm.x86.avx512.mask.cvtuqq2pd.128(<2 x i64> %x0, <2 x double> %x1, i8 -1) + %res2 = fadd <2 x double> %res, %res1 + ret <2 x double> %res2 +} + +declare <4 x double> @llvm.x86.avx512.mask.cvtuqq2pd.256(<4 x i64>, <4 x double>, i8) + +define <4 x double>@test_int_x86_avx512_mask_cvt_uqq2pd_256(<4 x i64> %x0, <4 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_uqq2pd_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtuqq2pd %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtuqq2pd %ymm0, %ymm0 +; CHECK-NEXT: vaddpd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.avx512.mask.cvtuqq2pd.256(<4 x i64> %x0, <4 x double> %x1, i8 %x2) + %res1 = call <4 x double> @llvm.x86.avx512.mask.cvtuqq2pd.256(<4 x i64> %x0, <4 x double> %x1, i8 -1) + %res2 = fadd <4 x double> %res, %res1 + ret <4 x double> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.128(<2 x i64>, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_cvt_uqq2ps_128(<2 x i64> %x0, <4 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_uqq2ps_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtuqq2ps %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtuqq2ps %xmm0, %xmm0 +; CHECK-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.128(<2 x i64> %x0, <4 x float> %x1, i8 %x2) + %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.128(<2 x i64> %x0, <4 x float> %x1, i8 -1) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.256(<4 x i64>, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_cvt_uqq2ps_256(<4 x i64> %x0, <4 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_uqq2ps_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvtuqq2ps %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtuqq2ps %ymm0, %xmm0 +; CHECK-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.256(<4 x i64> %x0, <4 x float> %x1, i8 %x2) + %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.256(<4 x i64> %x0, <4 x float> %x1, i8 -1) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <2 x i64> @llvm.x86.avx512.mask.cvttps2uqq.128(<4 x float>, <2 x i64>, i8) + +define <2 x i64>@test_int_x86_avx512_mask_cvtt_ps2uqq_128(<4 x float> %x0, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2uqq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvttps2uqq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvttps2uqq %xmm0, %xmm0 +; CHECK-NEXT: vpaddq %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x i64> @llvm.x86.avx512.mask.cvttps2uqq.128(<4 x float> %x0, <2 x i64> %x1, i8 %x2) + %res1 = call <2 x i64> @llvm.x86.avx512.mask.cvttps2uqq.128(<4 x float> %x0, <2 x i64> %x1, i8 -1) + %res2 = add <2 x i64> %res, %res1 + ret <2 x i64> %res2 +} + +declare <4 x i64> @llvm.x86.avx512.mask.cvttps2uqq.256(<4 x float>, <4 x i64>, i8) + +define <4 x i64>@test_int_x86_avx512_mask_cvtt_ps2uqq_256(<4 x float> %x0, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2uqq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vcvttps2uqq %xmm0, %ymm1 {%k1} +; CHECK-NEXT: vcvttps2uqq %xmm0, %ymm0 +; CHECK-NEXT: vpaddq %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.mask.cvttps2uqq.256(<4 x float> %x0, <4 x i64> %x1, i8 %x2) + %res1 = call <4 x i64> @llvm.x86.avx512.mask.cvttps2uqq.256(<4 x float> %x0, <4 x i64> %x1, i8 -1) + %res2 = add <4 x i64> %res, %res1 + ret <4 x i64> %res2 +} + +declare <2 x double> @llvm.x86.avx512.mask.reduce.pd.128(<2 x double>, i32, <2 x double>, i8) + +; CHECK-LABEL: @test_int_x86_avx512_mask_reduce_pd_128 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vreducepd {{.*}}{%k1} +; CHECK: vreducepd +define <2 x double>@test_int_x86_avx512_mask_reduce_pd_128(<2 x double> %x0, <2 x double> %x2, i8 %x3) { + %res = call <2 x double> @llvm.x86.avx512.mask.reduce.pd.128(<2 x double> %x0, i32 4, <2 x double> %x2, i8 %x3) + %res1 = call <2 x double> @llvm.x86.avx512.mask.reduce.pd.128(<2 x double> %x0, i32 8, <2 x double> %x2, i8 -1) + %res2 = fadd <2 x double> %res, %res1 + ret <2 x double> %res2 +} + +declare <4 x double> @llvm.x86.avx512.mask.reduce.pd.256(<4 x double>, i32, <4 x double>, i8) + +; CHECK-LABEL: @test_int_x86_avx512_mask_reduce_pd_256 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vreducepd {{.*}}{%k1} +; CHECK: vreducepd +define <4 x double>@test_int_x86_avx512_mask_reduce_pd_256(<4 x double> %x0, <4 x double> %x2, i8 %x3) { + %res = call <4 x double> @llvm.x86.avx512.mask.reduce.pd.256(<4 x double> %x0, i32 4, <4 x double> %x2, i8 %x3) + %res1 = call <4 x double> @llvm.x86.avx512.mask.reduce.pd.256(<4 x double> %x0, i32 0, <4 x double> %x2, i8 -1) + %res2 = fadd <4 x double> %res, %res1 + ret <4 x double> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.reduce.ps.128(<4 x float>, i32, <4 x float>, i8) +; CHECK-LABEL: @test_int_x86_avx512_mask_reduce_ps_128 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vreduceps {{.*}}{%k1} +; CHECK: vreduceps +define <4 x float>@test_int_x86_avx512_mask_reduce_ps_128(<4 x float> %x0, <4 x float> %x2, i8 %x3) { + %res = call <4 x float> @llvm.x86.avx512.mask.reduce.ps.128(<4 x float> %x0, i32 4, <4 x float> %x2, i8 %x3) + %res1 = call <4 x float> @llvm.x86.avx512.mask.reduce.ps.128(<4 x float> %x0, i32 88, <4 x float> %x2, i8 -1) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <8 x float> @llvm.x86.avx512.mask.reduce.ps.256(<8 x float>, i32, <8 x float>, i8) + +; CHECK-LABEL: @test_int_x86_avx512_mask_reduce_ps_256 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vreduceps {{.*}}{%k1} +; CHECK: vreduceps +define <8 x float>@test_int_x86_avx512_mask_reduce_ps_256(<8 x float> %x0, <8 x float> %x2, i8 %x3) { + %res = call <8 x float> @llvm.x86.avx512.mask.reduce.ps.256(<8 x float> %x0, i32 11, <8 x float> %x2, i8 %x3) + %res1 = call <8 x float> @llvm.x86.avx512.mask.reduce.ps.256(<8 x float> %x0, i32 11, <8 x float> %x2, i8 -1) + %res2 = fadd <8 x float> %res, %res1 + ret <8 x float> %res2 +} + +declare <2 x double> @llvm.x86.avx512.mask.range.pd.128(<2 x double>, <2 x double>, i32, <2 x double>, i8) + +; CHECK-LABEL: @test_int_x86_avx512_mask_range_pd_128 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vrangepd {{.*}}{%k1} +; CHECK: vrangepd +define <2 x double>@test_int_x86_avx512_mask_range_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 %x4) { + %res = call <2 x double> @llvm.x86.avx512.mask.range.pd.128(<2 x double> %x0, <2 x double> %x1, i32 4, <2 x double> %x3, i8 %x4) + %res1 = call <2 x double> @llvm.x86.avx512.mask.range.pd.128(<2 x double> %x0, <2 x double> %x1, i32 8, <2 x double> %x3, i8 -1) + %res2 = fadd <2 x double> %res, %res1 + ret <2 x double> %res2 +} + +declare <4 x double> @llvm.x86.avx512.mask.range.pd.256(<4 x double>, <4 x double>, i32, <4 x double>, i8) + +; CHECK-LABEL: @test_int_x86_avx512_mask_range_pd_256 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vrangepd {{.*}}{%k1} +; CHECK: vrangepd +define <4 x double>@test_int_x86_avx512_mask_range_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x3, i8 %x4) { + %res = call <4 x double> @llvm.x86.avx512.mask.range.pd.256(<4 x double> %x0, <4 x double> %x1, i32 4, <4 x double> %x3, i8 %x4) + %res1 = call <4 x double> @llvm.x86.avx512.mask.range.pd.256(<4 x double> %x0, <4 x double> %x1, i32 88, <4 x double> %x3, i8 -1) + %res2 = fadd <4 x double> %res, %res1 + ret <4 x double> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.range.ps.128(<4 x float>, <4 x float>, i32, <4 x float>, i8) + +; CHECK-LABEL: @test_int_x86_avx512_mask_range_ps_128 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vrangeps {{.*}}{%k1} +; CHECK: vrangeps +define <4 x float>@test_int_x86_avx512_mask_range_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 %x4) { + %res = call <4 x float> @llvm.x86.avx512.mask.range.ps.128(<4 x float> %x0, <4 x float> %x1, i32 4, <4 x float> %x3, i8 %x4) + %res1 = call <4 x float> @llvm.x86.avx512.mask.range.ps.128(<4 x float> %x0, <4 x float> %x1, i32 88, <4 x float> %x3, i8 -1) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <8 x float> @llvm.x86.avx512.mask.range.ps.256(<8 x float>, <8 x float>, i32, <8 x float>, i8) + +; CHECK-LABEL: @test_int_x86_avx512_mask_range_ps_256 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vrangeps {{.*}}{%k1} +; CHECK: vrangeps +define <8 x float>@test_int_x86_avx512_mask_range_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x3, i8 %x4) { + %res = call <8 x float> @llvm.x86.avx512.mask.range.ps.256(<8 x float> %x0, <8 x float> %x1, i32 4, <8 x float> %x3, i8 %x4) + %res1 = call <8 x float> @llvm.x86.avx512.mask.range.ps.256(<8 x float> %x0, <8 x float> %x1, i32 88, <8 x float> %x3, i8 -1) + %res2 = fadd <8 x float> %res, %res1 + ret <8 x float> %res2 +} + +declare <2 x double> @llvm.x86.avx512.mask.vextractf64x2.256(<4 x double>, i32, <2 x double>, i8) + +define <2 x double>@test_int_x86_avx512_mask_vextractf64x2_256(<4 x double> %x0, <2 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vextractf64x2_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vextractf64x2 $1, %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vextractf64x2 $1, %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vextractf64x2 $1, %ymm0, %xmm0 +; CHECK-NEXT: vaddpd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: vaddpd %xmm2, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.vextractf64x2.256(<4 x double> %x0,i32 1, <2 x double> %x2, i8 %x3) + %res2 = call <2 x double> @llvm.x86.avx512.mask.vextractf64x2.256(<4 x double> %x0,i32 1, <2 x double> zeroinitializer, i8 %x3) + %res1 = call <2 x double> @llvm.x86.avx512.mask.vextractf64x2.256(<4 x double> %x0,i32 1, <2 x double> zeroinitializer, i8 -1) + %res3 = fadd <2 x double> %res, %res1 + %res4 = fadd <2 x double> %res3, %res2 + ret <2 x double> %res4 +} + +declare <4 x double> @llvm.x86.avx512.mask.insertf64x2.256(<4 x double>, <2 x double>, i32, <4 x double>, i8) + +define <4 x double>@test_int_x86_avx512_mask_insertf64x2_256(<4 x double> %x0, <2 x double> %x1, <4 x double> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_insertf64x2_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vinsertf64x2 $1, %xmm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: vinsertf64x2 $1, %xmm1, %ymm0, %ymm3 {%k1} {z} +; CHECK-NEXT: vinsertf64x2 $1, %xmm1, %ymm0, %ymm0 +; CHECK-NEXT: vaddpd %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: vaddpd %ymm0, %ymm3, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.avx512.mask.insertf64x2.256(<4 x double> %x0, <2 x double> %x1, i32 1, <4 x double> %x3, i8 %x4) + %res1 = call <4 x double> @llvm.x86.avx512.mask.insertf64x2.256(<4 x double> %x0, <2 x double> %x1, i32 1, <4 x double> %x3, i8 -1) + %res2 = call <4 x double> @llvm.x86.avx512.mask.insertf64x2.256(<4 x double> %x0, <2 x double> %x1, i32 1, <4 x double> zeroinitializer, i8 %x4) + %res3 = fadd <4 x double> %res, %res1 + %res4 = fadd <4 x double> %res2, %res3 + ret <4 x double> %res4 +} + +declare <4 x i64> @llvm.x86.avx512.mask.inserti64x2.256(<4 x i64>, <2 x i64>, i32, <4 x i64>, i8) + +define <4 x i64>@test_int_x86_avx512_mask_inserti64x2_256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_inserti64x2_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vinserti64x2 $1, %xmm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: vinserti64x2 $1, %xmm1, %ymm0, %ymm3 {%k1} {z} +; CHECK-NEXT: vinserti64x2 $1, %xmm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddq %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: vpaddq %ymm3, %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.mask.inserti64x2.256(<4 x i64> %x0, <2 x i64> %x1, i32 1, <4 x i64> %x3, i8 %x4) + %res1 = call <4 x i64> @llvm.x86.avx512.mask.inserti64x2.256(<4 x i64> %x0, <2 x i64> %x1, i32 1, <4 x i64> %x3, i8 -1) + %res2 = call <4 x i64> @llvm.x86.avx512.mask.inserti64x2.256(<4 x i64> %x0, <2 x i64> %x1, i32 1, <4 x i64> zeroinitializer, i8 %x4) + %res3 = add <4 x i64> %res, %res1 + %res4 = add <4 x i64> %res3, %res2 + ret <4 x i64> %res4 +} + +declare i8 @llvm.x86.avx512.mask.fpclass.ps.128(<4 x float>, i32, i8) + +; CHECK-LABEL: @test_int_x86_avx512_mask_fpclass_ps_128 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vfpclassps +; CHECK: {%k1} +; CHECK: vfpclassps +; CHECK: kmovb %k0 +define i8 @test_int_x86_avx512_mask_fpclass_ps_128(<4 x float> %x0, i8 %x1) { + %res = call i8 @llvm.x86.avx512.mask.fpclass.ps.128(<4 x float> %x0, i32 2, i8 %x1) + %res1 = call i8 @llvm.x86.avx512.mask.fpclass.ps.128(<4 x float> %x0, i32 4, i8 -1) + %res2 = add i8 %res, %res1 + ret i8 %res2 +} + +declare i8 @llvm.x86.avx512.mask.fpclass.ps.256(<8 x float>, i32, i8) + +; CHECK-LABEL: @test_int_x86_avx512_mask_fpclass_ps_256 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vfpclassps +; CHECK: {%k1} +; CHECK: vfpclassps +; CHECK: kmovb %k0 +define i8 @test_int_x86_avx512_mask_fpclass_ps_256(<8 x float> %x0, i8 %x1) { + %res = call i8 @llvm.x86.avx512.mask.fpclass.ps.256(<8 x float> %x0, i32 2, i8 %x1) + %res1 = call i8 @llvm.x86.avx512.mask.fpclass.ps.256(<8 x float> %x0, i32 4, i8 -1) + %res2 = add i8 %res, %res1 + ret i8 %res2 +} + +declare i8 @llvm.x86.avx512.mask.fpclass.pd.128(<2 x double>, i32, i8) + +; CHECK-LABEL: @test_int_x86_avx512_mask_fpclass_pd_128 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vfpclasspd +; CHECK: {%k1} +; CHECK: vfpclasspd +; CHECK: kmovb %k0 +define i8 @test_int_x86_avx512_mask_fpclass_pd_128(<2 x double> %x0, i8 %x1) { + %res = call i8 @llvm.x86.avx512.mask.fpclass.pd.128(<2 x double> %x0, i32 4, i8 %x1) + %res1 = call i8 @llvm.x86.avx512.mask.fpclass.pd.128(<2 x double> %x0, i32 2, i8 -1) + %res2 = add i8 %res, %res1 + ret i8 %res2 +} + +declare i8 @llvm.x86.avx512.mask.fpclass.pd.256(<4 x double>, i32, i8) + +; CHECK-LABEL: @test_int_x86_avx512_mask_fpclass_pd_256 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vfpclasspd +; CHECK: {%k1} +; CHECK: vfpclasspd +; CHECK: kmovb %k0 +define i8 @test_int_x86_avx512_mask_fpclass_pd_256(<4 x double> %x0, i8 %x1) { + %res = call i8 @llvm.x86.avx512.mask.fpclass.pd.256(<4 x double> %x0, i32 2, i8 %x1) + %res1 = call i8 @llvm.x86.avx512.mask.fpclass.pd.256(<4 x double> %x0, i32 4, i8 -1) + %res2 = add i8 %res, %res1 + ret i8 %res2 +} + +declare <8 x float> @llvm.x86.avx512.mask.broadcastf32x2.256(<4 x float>, <8 x float>, i8) + +define <8 x float>@test_int_x86_avx512_mask_broadcastf32x2_256(<4 x float> %x0, <8 x float> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_broadcastf32x2_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vbroadcastf32x2 %xmm0, %ymm1 {%k1} +; CHECK-NEXT: vbroadcastf32x2 %xmm0, %ymm2 {%k1} {z} +; CHECK-NEXT: vbroadcastf32x2 %xmm0, %ymm0 +; CHECK-NEXT: vaddps %ymm2, %ymm1, %ymm1 +; CHECK-NEXT: vaddps %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.avx512.mask.broadcastf32x2.256(<4 x float> %x0, <8 x float> %x2, i8 %x3) + %res1 = call <8 x float> @llvm.x86.avx512.mask.broadcastf32x2.256(<4 x float> %x0, <8 x float> zeroinitializer, i8 %x3) + %res2 = call <8 x float> @llvm.x86.avx512.mask.broadcastf32x2.256(<4 x float> %x0, <8 x float> %x2, i8 -1) + %res3 = fadd <8 x float> %res, %res1 + %res4 = fadd <8 x float> %res3, %res2 + ret <8 x float> %res4 +} + +declare <8 x i32> @llvm.x86.avx512.mask.broadcasti32x2.256(<4 x i32>, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_mask_broadcasti32x2_256(<4 x i32> %x0, <8 x i32> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_broadcasti32x2_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vbroadcasti32x2 %xmm0, %ymm1 {%k1} +; CHECK-NEXT: vbroadcasti32x2 %xmm0, %ymm2 {%k1} {z} +; CHECK-NEXT: vbroadcasti32x2 %xmm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm2, %ymm1, %ymm1 +; CHECK-NEXT: vpaddd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.mask.broadcasti32x2.256(<4 x i32> %x0, <8 x i32> %x2, i8 %x3) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.broadcasti32x2.256(<4 x i32> %x0, <8 x i32> zeroinitializer, i8 %x3) + %res2 = call <8 x i32> @llvm.x86.avx512.mask.broadcasti32x2.256(<4 x i32> %x0, <8 x i32> %x2, i8 -1) + %res3 = add <8 x i32> %res, %res1 + %res4 = add <8 x i32> %res3, %res2 + ret <8 x i32> %res4 +} + +declare <4 x i32> @llvm.x86.avx512.mask.broadcasti32x2.128(<4 x i32>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_broadcasti32x2_128(<4 x i32> %x0, <4 x i32> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_broadcasti32x2_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k1 +; CHECK-NEXT: vbroadcasti32x2 %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vbroadcasti32x2 %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vbroadcasti32x2 %xmm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm2, %xmm1, %xmm1 +; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.broadcasti32x2.128(<4 x i32> %x0, <4 x i32> %x2, i8 %x3) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.broadcasti32x2.128(<4 x i32> %x0, <4 x i32> zeroinitializer, i8 %x3) + %res2 = call <4 x i32> @llvm.x86.avx512.mask.broadcasti32x2.128(<4 x i32> %x0, <4 x i32> %x2, i8 -1) + %res3 = add <4 x i32> %res, %res1 + %res4 = add <4 x i32> %res3, %res2 + ret <4 x i32> %res4 +} + +declare i8 @llvm.x86.avx512.cvtd2mask.128(<4 x i32>) + +define i8@test_int_x86_avx512_cvtd2mask_128(<4 x i32> %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtd2mask_128: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovd2m %xmm0, %k0 +; CHECK-NEXT: kmovb %k0, %eax +; CHECK-NEXT: retq + %res = call i8 @llvm.x86.avx512.cvtd2mask.128(<4 x i32> %x0) + ret i8 %res +} + +declare i8 @llvm.x86.avx512.cvtd2mask.256(<8 x i32>) + +define i8@test_int_x86_avx512_cvtd2mask_256(<8 x i32> %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtd2mask_256: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovd2m %ymm0, %k0 +; CHECK-NEXT: kmovb %k0, %eax +; CHECK-NEXT: retq + %res = call i8 @llvm.x86.avx512.cvtd2mask.256(<8 x i32> %x0) + ret i8 %res +} + +declare i8 @llvm.x86.avx512.cvtq2mask.128(<2 x i64>) + +define i8@test_int_x86_avx512_cvtq2mask_128(<2 x i64> %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtq2mask_128: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovq2m %xmm0, %k0 +; CHECK-NEXT: kmovb %k0, %eax +; CHECK-NEXT: retq + %res = call i8 @llvm.x86.avx512.cvtq2mask.128(<2 x i64> %x0) + ret i8 %res +} + +declare i8 @llvm.x86.avx512.cvtq2mask.256(<4 x i64>) + +define i8@test_int_x86_avx512_cvtq2mask_256(<4 x i64> %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtq2mask_256: +; CHECK: ## BB#0: +; CHECK-NEXT: vpmovq2m %ymm0, %k0 +; CHECK-NEXT: kmovb %k0, %eax +; CHECK-NEXT: retq + %res = call i8 @llvm.x86.avx512.cvtq2mask.256(<4 x i64> %x0) + ret i8 %res +} + +declare <4 x i32> @llvm.x86.avx512.cvtmask2d.128(i8) + +define <4 x i32>@test_int_x86_avx512_cvtmask2d_128(i8 %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtmask2d_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k0 +; CHECK-NEXT: vpmovm2d %k0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.cvtmask2d.128(i8 %x0) + ret <4 x i32> %res +} + +declare <8 x i32> @llvm.x86.avx512.cvtmask2d.256(i8) + +define <8 x i32>@test_int_x86_avx512_cvtmask2d_256(i8 %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtmask2d_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k0 +; CHECK-NEXT: vpmovm2d %k0, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.cvtmask2d.256(i8 %x0) + ret <8 x i32> %res +} + +declare <2 x i64> @llvm.x86.avx512.cvtmask2q.128(i8) + +define <2 x i64>@test_int_x86_avx512_cvtmask2q_128(i8 %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtmask2q_128: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k0 +; CHECK-NEXT: vpmovm2q %k0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x i64> @llvm.x86.avx512.cvtmask2q.128(i8 %x0) + ret <2 x i64> %res +} + +declare <4 x i64> @llvm.x86.avx512.cvtmask2q.256(i8) + +define <4 x i64>@test_int_x86_avx512_cvtmask2q_256(i8 %x0) { +; CHECK-LABEL: test_int_x86_avx512_cvtmask2q_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovb %edi, %k0 +; CHECK-NEXT: vpmovm2q %k0, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.cvtmask2q.256(i8 %x0) + ret <4 x i64> %res +} +declare <4 x double> @llvm.x86.avx512.mask.broadcastf64x2.256(<2 x double>, <4 x double>, i8) + +define <4 x double>@test_int_x86_avx512_mask_broadcastf64x2_256(<2 x double> %x0, <4 x double> %x2, i8 %mask) { +; CHECK-LABEL: test_int_x86_avx512_mask_broadcastf64x2_256: +; CHECK: kmovb %edi, %k1 +; CHECK: vshuff64x2 $0, %ymm0, %ymm0, %ymm2 {%k1} {z} +; CHECK: vshuff64x2 $0, %ymm0, %ymm0, %ymm1 {%k1} +; CHECK: vshuff64x2 $0, %ymm0, %ymm0, %ymm0 +; CHECK: vaddpd %ymm1, %ymm0, %ymm0 +; CHECK: vaddpd %ymm0, %ymm2, %ymm0 + + %res1 = call <4 x double> @llvm.x86.avx512.mask.broadcastf64x2.256(<2 x double> %x0, <4 x double> %x2, i8 -1) + %res2 = call <4 x double> @llvm.x86.avx512.mask.broadcastf64x2.256(<2 x double> %x0, <4 x double> %x2, i8 %mask) + %res3 = call <4 x double> @llvm.x86.avx512.mask.broadcastf64x2.256(<2 x double> %x0, <4 x double> zeroinitializer, i8 %mask) + %res4 = fadd <4 x double> %res1, %res2 + %res5 = fadd <4 x double> %res3, %res4 + ret <4 x double> %res5 +} + +declare <4 x i64> @llvm.x86.avx512.mask.broadcasti64x2.256(<2 x i64>, <4 x i64>, i8) + +define <4 x i64>@test_int_x86_avx512_mask_broadcasti64x2_256(<2 x i64> %x0, <4 x i64> %x2, i8 %mask) { +; CHECK-LABEL: test_int_x86_avx512_mask_broadcasti64x2_256: +; CHECK: kmovb %edi, %k1 +; CHECK: vshufi64x2 $0, %ymm0, %ymm0, %ymm2 {%k1} {z} +; CHECK: vshufi64x2 $0, %ymm0, %ymm0, %ymm1 {%k1} +; CHECK: vshufi64x2 $0, %ymm0, %ymm0, %ymm0 +; CHECK: vpaddq %ymm1, %ymm0, %ymm0 +; CHECK: vpaddq %ymm0, %ymm2, %ymm0 + + %res1 = call <4 x i64> @llvm.x86.avx512.mask.broadcasti64x2.256(<2 x i64> %x0, <4 x i64> %x2, i8 -1) + %res2 = call <4 x i64> @llvm.x86.avx512.mask.broadcasti64x2.256(<2 x i64> %x0, <4 x i64> %x2, i8 %mask) + %res3 = call <4 x i64> @llvm.x86.avx512.mask.broadcasti64x2.256(<2 x i64> %x0, <4 x i64> zeroinitializer, i8 %mask) + %res4 = add <4 x i64> %res1, %res2 + %res5 = add <4 x i64> %res3, %res4 + ret <4 x i64> %res5 +} diff --git a/test/CodeGen/X86/avx512vl-intrinsics.ll b/test/CodeGen/X86/avx512vl-intrinsics.ll index fb7c93dc53b3..d9e8728c5ca6 100644 --- a/test/CodeGen/X86/avx512vl-intrinsics.ll +++ b/test/CodeGen/X86/avx512vl-intrinsics.ll @@ -1867,7 +1867,7 @@ define <4 x i32> @test_mask_xor_epi32_rmbk_128(<4 x i32> %a, i32* %ptr_b, <4 x i define <4 x i32> @test_mask_xor_epi32_rmbkz_128(<4 x i32> %a, i32* %ptr_b, i8 %mask) { ;CHECK-LABEL: test_mask_xor_epi32_rmbkz_128 - ;CHECK: vpxord (%rdi){1to4}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x99,0xef,0x07] + ;CHECK: vpxord (%rdi){1to4}, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x99,0xef,0x07] %q = load i32, i32* %ptr_b %vecinit.i = insertelement <4 x i32> undef, i32 %q, i32 0 %b = shufflevector <4 x i32> %vecinit.i, <4 x i32> undef, <4 x i32> zeroinitializer @@ -2299,7 +2299,7 @@ define <8 x float> @test_mm512_maskz_add_ps_256(<8 x float> %a0, <8 x float> %a1 define <8 x float> @test_mm512_mask_add_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) { ;CHECK-LABEL: test_mm512_mask_add_ps_256 - ;CHECK: vaddps %ymm1, %ymm0, %ymm2 {%k1} + ;CHECK: vaddps %ymm1, %ymm0, %ymm2 {%k1} %res = call <8 x float> @llvm.x86.avx512.mask.add.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) ret <8 x float> %res } @@ -2321,7 +2321,7 @@ define <4 x float> @test_mm512_maskz_add_ps_128(<4 x float> %a0, <4 x float> %a1 define <4 x float> @test_mm512_mask_add_ps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) { ;CHECK-LABEL: test_mm512_mask_add_ps_128 - ;CHECK: vaddps %xmm1, %xmm0, %xmm2 {%k1} + ;CHECK: vaddps %xmm1, %xmm0, %xmm2 {%k1} %res = call <4 x float> @llvm.x86.avx512.mask.add.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) ret <4 x float> %res } @@ -2343,7 +2343,7 @@ define <8 x float> @test_mm512_maskz_sub_ps_256(<8 x float> %a0, <8 x float> %a1 define <8 x float> @test_mm512_mask_sub_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) { ;CHECK-LABEL: test_mm512_mask_sub_ps_256 - ;CHECK: vsubps %ymm1, %ymm0, %ymm2 {%k1} + ;CHECK: vsubps %ymm1, %ymm0, %ymm2 {%k1} %res = call <8 x float> @llvm.x86.avx512.mask.sub.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) ret <8 x float> %res } @@ -2365,7 +2365,7 @@ define <4 x float> @test_mm512_maskz_sub_ps_128(<4 x float> %a0, <4 x float> %a1 define <4 x float> @test_mm512_mask_sub_ps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) { ;CHECK-LABEL: test_mm512_mask_sub_ps_128 - ;CHECK: vsubps %xmm1, %xmm0, %xmm2 {%k1} + ;CHECK: vsubps %xmm1, %xmm0, %xmm2 {%k1} %res = call <4 x float> @llvm.x86.avx512.mask.sub.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) ret <4 x float> %res } @@ -2387,7 +2387,7 @@ define <8 x float> @test_mm512_maskz_mul_ps_256(<8 x float> %a0, <8 x float> %a1 define <8 x float> @test_mm512_mask_mul_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) { ;CHECK-LABEL: test_mm512_mask_mul_ps_256 - ;CHECK: vmulps %ymm1, %ymm0, %ymm2 {%k1} + ;CHECK: vmulps %ymm1, %ymm0, %ymm2 {%k1} %res = call <8 x float> @llvm.x86.avx512.mask.mul.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) ret <8 x float> %res } @@ -2409,7 +2409,7 @@ define <4 x float> @test_mm512_maskz_mul_ps_128(<4 x float> %a0, <4 x float> %a1 define <4 x float> @test_mm512_mask_mul_ps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) { ;CHECK-LABEL: test_mm512_mask_mul_ps_128 - ;CHECK: vmulps %xmm1, %xmm0, %xmm2 {%k1} + ;CHECK: vmulps %xmm1, %xmm0, %xmm2 {%k1} %res = call <4 x float> @llvm.x86.avx512.mask.mul.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) ret <4 x float> %res } @@ -2431,7 +2431,7 @@ define <8 x float> @test_mm512_maskz_div_ps_256(<8 x float> %a0, <8 x float> %a1 define <8 x float> @test_mm512_mask_div_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) { ;CHECK-LABEL: test_mm512_mask_div_ps_256 - ;CHECK: vdivps %ymm1, %ymm0, %ymm2 {%k1} + ;CHECK: vdivps %ymm1, %ymm0, %ymm2 {%k1} %res = call <8 x float> @llvm.x86.avx512.mask.div.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) ret <8 x float> %res } @@ -2453,7 +2453,7 @@ define <4 x float> @test_mm512_maskz_div_ps_128(<4 x float> %a0, <4 x float> %a1 define <4 x float> @test_mm512_mask_div_ps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) { ;CHECK-LABEL: test_mm512_mask_div_ps_128 - ;CHECK: vdivps %xmm1, %xmm0, %xmm2 {%k1} + ;CHECK: vdivps %xmm1, %xmm0, %xmm2 {%k1} %res = call <4 x float> @llvm.x86.avx512.mask.div.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) ret <4 x float> %res } @@ -2475,7 +2475,7 @@ define <8 x float> @test_mm512_maskz_max_ps_256(<8 x float> %a0, <8 x float> %a1 define <8 x float> @test_mm512_mask_max_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) { ;CHECK-LABEL: test_mm512_mask_max_ps_256 - ;CHECK: vmaxps %ymm1, %ymm0, %ymm2 {%k1} + ;CHECK: vmaxps %ymm1, %ymm0, %ymm2 {%k1} %res = call <8 x float> @llvm.x86.avx512.mask.max.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) ret <8 x float> %res } @@ -2497,7 +2497,7 @@ define <4 x float> @test_mm512_maskz_max_ps_128(<4 x float> %a0, <4 x float> %a1 define <4 x float> @test_mm512_mask_max_ps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) { ;CHECK-LABEL: test_mm512_mask_max_ps_128 - ;CHECK: vmaxps %xmm1, %xmm0, %xmm2 {%k1} + ;CHECK: vmaxps %xmm1, %xmm0, %xmm2 {%k1} %res = call <4 x float> @llvm.x86.avx512.mask.max.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) ret <4 x float> %res } @@ -2519,7 +2519,7 @@ define <8 x float> @test_mm512_maskz_min_ps_256(<8 x float> %a0, <8 x float> %a1 define <8 x float> @test_mm512_mask_min_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) { ;CHECK-LABEL: test_mm512_mask_min_ps_256 - ;CHECK: vminps %ymm1, %ymm0, %ymm2 {%k1} + ;CHECK: vminps %ymm1, %ymm0, %ymm2 {%k1} %res = call <8 x float> @llvm.x86.avx512.mask.min.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %src, i8 %mask) ret <8 x float> %res } @@ -2541,7 +2541,7 @@ define <4 x float> @test_mm512_maskz_min_ps_128(<4 x float> %a0, <4 x float> %a1 define <4 x float> @test_mm512_mask_min_ps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) { ;CHECK-LABEL: test_mm512_mask_min_ps_128 - ;CHECK: vminps %xmm1, %xmm0, %xmm2 {%k1} + ;CHECK: vminps %xmm1, %xmm0, %xmm2 {%k1} %res = call <4 x float> @llvm.x86.avx512.mask.min.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %src, i8 %mask) ret <4 x float> %res } @@ -2591,9 +2591,9 @@ declare <8 x float> @llvm.x86.avx512.mask.getexp.ps.256(<8 x float>, <8 x float> declare <4 x i32> @llvm.x86.avx512.mask.pmaxs.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_d_128 -; CHECK-NOT: call +; CHECK-NOT: call ; CHECK: vpmaxsd %xmm -; CHECK: {%k1} +; CHECK: {%k1} define <4 x i32>@test_int_x86_avx512_mask_pmaxs_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %mask) { %res = call <4 x i32> @llvm.x86.avx512.mask.pmaxs.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2 ,i8 %mask) %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmaxs.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %mask) @@ -2604,9 +2604,9 @@ define <4 x i32>@test_int_x86_avx512_mask_pmaxs_d_128(<4 x i32> %x0, <4 x i32> % declare <8 x i32> @llvm.x86.avx512.mask.pmaxs.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_d_256 -; CHECK-NOT: call +; CHECK-NOT: call ; CHECK: vpmaxsd %ymm -; CHECK: {%k1} +; CHECK: {%k1} define <8 x i32>@test_int_x86_avx512_mask_pmaxs_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) { %res = call <8 x i32> @llvm.x86.avx512.mask.pmaxs.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmaxs.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1) @@ -2617,9 +2617,9 @@ define <8 x i32>@test_int_x86_avx512_mask_pmaxs_d_256(<8 x i32> %x0, <8 x i32> % declare <2 x i64> @llvm.x86.avx512.mask.pmaxs.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_q_128 -; CHECK-NOT: call +; CHECK-NOT: call ; CHECK: vpmaxsq %xmm -; CHECK: {%k1} +; CHECK: {%k1} define <2 x i64>@test_int_x86_avx512_mask_pmaxs_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) { %res = call <2 x i64> @llvm.x86.avx512.mask.pmaxs.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) %res1 = call <2 x i64> @llvm.x86.avx512.mask.pmaxs.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1) @@ -2630,9 +2630,9 @@ define <2 x i64>@test_int_x86_avx512_mask_pmaxs_q_128(<2 x i64> %x0, <2 x i64> % declare <4 x i64> @llvm.x86.avx512.mask.pmaxs.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxs_q_256 -; CHECK-NOT: call +; CHECK-NOT: call ; CHECK: vpmaxsq %ymm -; CHECK: {%k1} +; CHECK: {%k1} define <4 x i64>@test_int_x86_avx512_mask_pmaxs_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %mask) { %res = call <4 x i64> @llvm.x86.avx512.mask.pmaxs.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %mask) %res1 = call <4 x i64> @llvm.x86.avx512.mask.pmaxs.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %mask) @@ -2643,9 +2643,9 @@ define <4 x i64>@test_int_x86_avx512_mask_pmaxs_q_256(<4 x i64> %x0, <4 x i64> % declare <4 x i32> @llvm.x86.avx512.mask.pmaxu.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_d_128 -; CHECK-NOT: call +; CHECK-NOT: call ; CHECK: vpmaxud %xmm -; CHECK: {%k1} +; CHECK: {%k1} define <4 x i32>@test_int_x86_avx512_mask_pmaxu_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2,i8 %mask) { %res = call <4 x i32> @llvm.x86.avx512.mask.pmaxu.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %mask) %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmaxu.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %mask) @@ -2656,9 +2656,9 @@ define <4 x i32>@test_int_x86_avx512_mask_pmaxu_d_128(<4 x i32> %x0, <4 x i32> % declare <8 x i32> @llvm.x86.avx512.mask.pmaxu.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_d_256 -; CHECK-NOT: call +; CHECK-NOT: call ; CHECK: vpmaxud %ymm -; CHECK: {%k1} +; CHECK: {%k1} define <8 x i32>@test_int_x86_avx512_mask_pmaxu_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) { %res = call <8 x i32> @llvm.x86.avx512.mask.pmaxu.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmaxu.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1) @@ -2669,9 +2669,9 @@ define <8 x i32>@test_int_x86_avx512_mask_pmaxu_d_256(<8 x i32> %x0, <8 x i32> % declare <2 x i64> @llvm.x86.avx512.mask.pmaxu.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_q_128 -; CHECK-NOT: call +; CHECK-NOT: call ; CHECK: vpmaxuq %xmm -; CHECK: {%k1} +; CHECK: {%k1} define <2 x i64>@test_int_x86_avx512_mask_pmaxu_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) { %res = call <2 x i64> @llvm.x86.avx512.mask.pmaxu.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) %res1 = call <2 x i64> @llvm.x86.avx512.mask.pmaxu.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1) @@ -2682,9 +2682,9 @@ define <2 x i64>@test_int_x86_avx512_mask_pmaxu_q_128(<2 x i64> %x0, <2 x i64> % declare <4 x i64> @llvm.x86.avx512.mask.pmaxu.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pmaxu_q_256 -; CHECK-NOT: call +; CHECK-NOT: call ; CHECK: vpmaxuq %ymm -; CHECK: {%k1} +; CHECK: {%k1} define <4 x i64>@test_int_x86_avx512_mask_pmaxu_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %mask) { %res = call <4 x i64> @llvm.x86.avx512.mask.pmaxu.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %mask) %res1 = call <4 x i64> @llvm.x86.avx512.mask.pmaxu.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %mask) @@ -2695,9 +2695,9 @@ define <4 x i64>@test_int_x86_avx512_mask_pmaxu_q_256(<4 x i64> %x0, <4 x i64> % declare <4 x i32> @llvm.x86.avx512.mask.pmins.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_d_128 -; CHECK-NOT: call +; CHECK-NOT: call ; CHECK: vpminsd %xmm -; CHECK: {%k1} +; CHECK: {%k1} define <4 x i32>@test_int_x86_avx512_mask_pmins_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %mask) { %res = call <4 x i32> @llvm.x86.avx512.mask.pmins.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %mask) %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmins.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %mask) @@ -2708,9 +2708,9 @@ define <4 x i32>@test_int_x86_avx512_mask_pmins_d_128(<4 x i32> %x0, <4 x i32> % declare <8 x i32> @llvm.x86.avx512.mask.pmins.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_d_256 -; CHECK-NOT: call +; CHECK-NOT: call ; CHECK: vpminsd %ymm -; CHECK: {%k1} +; CHECK: {%k1} define <8 x i32>@test_int_x86_avx512_mask_pmins_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) { %res = call <8 x i32> @llvm.x86.avx512.mask.pmins.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmins.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1) @@ -2721,9 +2721,9 @@ define <8 x i32>@test_int_x86_avx512_mask_pmins_d_256(<8 x i32> %x0, <8 x i32> % declare <2 x i64> @llvm.x86.avx512.mask.pmins.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_q_128 -; CHECK-NOT: call +; CHECK-NOT: call ; CHECK: vpminsq %xmm -; CHECK: {%k1} +; CHECK: {%k1} define <2 x i64>@test_int_x86_avx512_mask_pmins_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) { %res = call <2 x i64> @llvm.x86.avx512.mask.pmins.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) %res1 = call <2 x i64> @llvm.x86.avx512.mask.pmins.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1) @@ -2734,9 +2734,9 @@ define <2 x i64>@test_int_x86_avx512_mask_pmins_q_128(<2 x i64> %x0, <2 x i64> % declare <4 x i64> @llvm.x86.avx512.mask.pmins.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pmins_q_256 -; CHECK-NOT: call +; CHECK-NOT: call ; CHECK: vpminsq %ymm -; CHECK: {%k1} +; CHECK: {%k1} define <4 x i64>@test_int_x86_avx512_mask_pmins_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %mask) { %res = call <4 x i64> @llvm.x86.avx512.mask.pmins.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %mask) %res1 = call <4 x i64> @llvm.x86.avx512.mask.pmins.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %mask) @@ -2747,9 +2747,9 @@ define <4 x i64>@test_int_x86_avx512_mask_pmins_q_256(<4 x i64> %x0, <4 x i64> % declare <4 x i32> @llvm.x86.avx512.mask.pminu.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_d_128 -; CHECK-NOT: call +; CHECK-NOT: call ; CHECK: vpminud %xmm -; CHECK: {%k1} +; CHECK: {%k1} define <4 x i32>@test_int_x86_avx512_mask_pminu_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %mask) { %res = call <4 x i32> @llvm.x86.avx512.mask.pminu.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %mask) %res1 = call <4 x i32> @llvm.x86.avx512.mask.pminu.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %mask) @@ -2760,9 +2760,9 @@ define <4 x i32>@test_int_x86_avx512_mask_pminu_d_128(<4 x i32> %x0, <4 x i32> % declare <8 x i32> @llvm.x86.avx512.mask.pminu.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_d_256 -; CHECK-NOT: call +; CHECK-NOT: call ; CHECK: vpminud %ymm -; CHECK: {%k1} +; CHECK: {%k1} define <8 x i32>@test_int_x86_avx512_mask_pminu_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) { %res = call <8 x i32> @llvm.x86.avx512.mask.pminu.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) %res1 = call <8 x i32> @llvm.x86.avx512.mask.pminu.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1) @@ -2773,9 +2773,9 @@ define <8 x i32>@test_int_x86_avx512_mask_pminu_d_256(<8 x i32> %x0, <8 x i32> % declare <2 x i64> @llvm.x86.avx512.mask.pminu.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_q_128 -; CHECK-NOT: call +; CHECK-NOT: call ; CHECK: vpminuq %xmm -; CHECK: {%k1} +; CHECK: {%k1} define <2 x i64>@test_int_x86_avx512_mask_pminu_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) { %res = call <2 x i64> @llvm.x86.avx512.mask.pminu.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) %res1 = call <2 x i64> @llvm.x86.avx512.mask.pminu.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1) @@ -2786,9 +2786,9 @@ define <2 x i64>@test_int_x86_avx512_mask_pminu_q_128(<2 x i64> %x0, <2 x i64> % declare <4 x i64> @llvm.x86.avx512.mask.pminu.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pminu_q_256 -; CHECK-NOT: call +; CHECK-NOT: call ; CHECK: vpminuq %ymm -; CHECK: {%k1} +; CHECK: {%k1} define <4 x i64>@test_int_x86_avx512_mask_pminu_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %mask) { %res = call <4 x i64> @llvm.x86.avx512.mask.pminu.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %mask) %res1 = call <4 x i64> @llvm.x86.avx512.mask.pminu.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %mask) @@ -2799,8 +2799,8 @@ define <4 x i64>@test_int_x86_avx512_mask_pminu_q_256(<4 x i64> %x0, <4 x i64> % declare <4 x i32> @llvm.x86.avx512.mask.vpermt2var.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_d_128 -; CHECK-NOT: call -; CHECK: kmov +; CHECK-NOT: call +; CHECK: kmov ; CHECK: vpermt2d %xmm{{.*}}{%k1} ; CHECK-NOT: {z} define <4 x i32>@test_int_x86_avx512_mask_vpermt2var_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) { @@ -2813,8 +2813,8 @@ define <4 x i32>@test_int_x86_avx512_mask_vpermt2var_d_128(<4 x i32> %x0, <4 x i declare <4 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8) ; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_d_128 -; CHECK-NOT: call -; CHECK: kmov +; CHECK-NOT: call +; CHECK: kmov ; CHECK: vpermt2d %xmm{{.*}}{%k1} {z} define <4 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) { %res = call <4 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) @@ -2826,8 +2826,8 @@ define <4 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_128(<4 x i32> %x0, <4 x declare <8 x i32> @llvm.x86.avx512.mask.vpermt2var.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermt2var_d_256 -; CHECK-NOT: call -; CHECK: kmov +; CHECK-NOT: call +; CHECK: kmov ; CHECK: vpermt2d %ymm{{.*}}{%k1} ; CHECK-NOT: {z} define <8 x i32>@test_int_x86_avx512_mask_vpermt2var_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) { @@ -2840,8 +2840,8 @@ define <8 x i32>@test_int_x86_avx512_mask_vpermt2var_d_256(<8 x i32> %x0, <8 x i declare <8 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8) ; CHECK-LABEL: @test_int_x86_avx512_maskz_vpermt2var_d_256 -; CHECK-NOT: call -; CHECK: kmov +; CHECK-NOT: call +; CHECK: kmov ; CHECK: vpermt2d {{.*}}{%k1} {z} define <8 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) { %res = call <8 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) @@ -2853,9 +2853,9 @@ define <8 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_256(<8 x i32> %x0, <8 x declare <2 x double> @llvm.x86.avx512.mask.vpermi2var.pd.128(<2 x double>, <2 x i64>, <2 x double>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_pd_128 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpermi2pd %xmm{{.*}}{%k1} +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vpermi2pd %xmm{{.*}}{%k1} define <2 x double>@test_int_x86_avx512_mask_vpermi2var_pd_128(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2, i8 %x3) { %res = call <2 x double> @llvm.x86.avx512.mask.vpermi2var.pd.128(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2, i8 %x3) %res1 = call <2 x double> @llvm.x86.avx512.mask.vpermi2var.pd.128(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2, i8 -1) @@ -2866,9 +2866,9 @@ define <2 x double>@test_int_x86_avx512_mask_vpermi2var_pd_128(<2 x double> %x0, declare <4 x double> @llvm.x86.avx512.mask.vpermi2var.pd.256(<4 x double>, <4 x i64>, <4 x double>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_pd_256 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpermi2pd %ymm{{.*}}{%k1} +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vpermi2pd %ymm{{.*}}{%k1} define <4 x double>@test_int_x86_avx512_mask_vpermi2var_pd_256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2, i8 %x3) { %res = call <4 x double> @llvm.x86.avx512.mask.vpermi2var.pd.256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2, i8 %x3) %res1 = call <4 x double> @llvm.x86.avx512.mask.vpermi2var.pd.256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2, i8 -1) @@ -2879,9 +2879,9 @@ define <4 x double>@test_int_x86_avx512_mask_vpermi2var_pd_256(<4 x double> %x0, declare <4 x float> @llvm.x86.avx512.mask.vpermi2var.ps.128(<4 x float>, <4 x i32>, <4 x float>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_ps_128 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpermi2ps %xmm{{.*}}{%k1} +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vpermi2ps %xmm{{.*}}{%k1} define <4 x float>@test_int_x86_avx512_mask_vpermi2var_ps_128(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2, i8 %x3) { %res = call <4 x float> @llvm.x86.avx512.mask.vpermi2var.ps.128(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2, i8 %x3) %res1 = call <4 x float> @llvm.x86.avx512.mask.vpermi2var.ps.128(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2, i8 -1) @@ -2892,9 +2892,9 @@ define <4 x float>@test_int_x86_avx512_mask_vpermi2var_ps_128(<4 x float> %x0, < declare <8 x float> @llvm.x86.avx512.mask.vpermi2var.ps.256(<8 x float>, <8 x i32>, <8 x float>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_vpermi2var_ps_256 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpermi2ps %ymm{{.*}}{%k1} +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vpermi2ps %ymm{{.*}}{%k1} define <8 x float>@test_int_x86_avx512_mask_vpermi2var_ps_256(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2, i8 %x3) { %res = call <8 x float> @llvm.x86.avx512.mask.vpermi2var.ps.256(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2, i8 %x3) %res1 = call <8 x float> @llvm.x86.avx512.mask.vpermi2var.ps.256(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2, i8 -1) @@ -2905,9 +2905,9 @@ define <8 x float>@test_int_x86_avx512_mask_vpermi2var_ps_256(<8 x float> %x0, < declare <2 x i64> @llvm.x86.avx512.mask.pabs.q.128(<2 x i64>, <2 x i64>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_q_128 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpabsq{{.*}}{%k1} +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vpabsq{{.*}}{%k1} define <2 x i64>@test_int_x86_avx512_mask_pabs_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) { %res = call <2 x i64> @llvm.x86.avx512.mask.pabs.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) %res1 = call <2 x i64> @llvm.x86.avx512.mask.pabs.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 -1) @@ -2918,9 +2918,9 @@ define <2 x i64>@test_int_x86_avx512_mask_pabs_q_128(<2 x i64> %x0, <2 x i64> %x declare <4 x i64> @llvm.x86.avx512.mask.pabs.q.256(<4 x i64>, <4 x i64>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_q_256 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpabsq{{.*}}{%k1} +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vpabsq{{.*}}{%k1} define <4 x i64>@test_int_x86_avx512_mask_pabs_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) { %res = call <4 x i64> @llvm.x86.avx512.mask.pabs.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) %res1 = call <4 x i64> @llvm.x86.avx512.mask.pabs.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 -1) @@ -2931,9 +2931,9 @@ define <4 x i64>@test_int_x86_avx512_mask_pabs_q_256(<4 x i64> %x0, <4 x i64> %x declare <4 x i32> @llvm.x86.avx512.mask.pabs.d.128(<4 x i32>, <4 x i32>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_d_128 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpabsd{{.*}}{%k1} +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vpabsd{{.*}}{%k1} define <4 x i32>@test_int_x86_avx512_mask_pabs_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) { %res = call <4 x i32> @llvm.x86.avx512.mask.pabs.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) %res1 = call <4 x i32> @llvm.x86.avx512.mask.pabs.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 -1) @@ -2944,9 +2944,9 @@ define <4 x i32>@test_int_x86_avx512_mask_pabs_d_128(<4 x i32> %x0, <4 x i32> %x declare <8 x i32> @llvm.x86.avx512.mask.pabs.d.256(<8 x i32>, <8 x i32>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_pabs_d_256 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vpabsd{{.*}}{%k1} +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vpabsd{{.*}}{%k1} define <8 x i32>@test_int_x86_avx512_mask_pabs_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) { %res = call <8 x i32> @llvm.x86.avx512.mask.pabs.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) %res1 = call <8 x i32> @llvm.x86.avx512.mask.pabs.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 -1) @@ -2958,9 +2958,9 @@ define <8 x i32>@test_int_x86_avx512_mask_pabs_d_256(<8 x i32> %x0, <8 x i32> %x declare <2 x double> @llvm.x86.avx512.mask.scalef.pd.128(<2 x double>, <2 x double>, <2 x double>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_scalef_pd_128 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vscalefpd{{.*}}{%k1} +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vscalefpd{{.*}}{%k1} define <2 x double>@test_int_x86_avx512_mask_scalef_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) { %res = call <2 x double> @llvm.x86.avx512.mask.scalef.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) %res1 = call <2 x double> @llvm.x86.avx512.mask.scalef.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1) @@ -2971,9 +2971,9 @@ define <2 x double>@test_int_x86_avx512_mask_scalef_pd_128(<2 x double> %x0, <2 declare <4 x double> @llvm.x86.avx512.mask.scalef.pd.256(<4 x double>, <4 x double>, <4 x double>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_scalef_pd_256 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vscalefpd{{.*}}{%k1} +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vscalefpd{{.*}}{%k1} define <4 x double>@test_int_x86_avx512_mask_scalef_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) { %res = call <4 x double> @llvm.x86.avx512.mask.scalef.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) %res1 = call <4 x double> @llvm.x86.avx512.mask.scalef.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1) @@ -2983,9 +2983,9 @@ define <4 x double>@test_int_x86_avx512_mask_scalef_pd_256(<4 x double> %x0, <4 declare <4 x float> @llvm.x86.avx512.mask.scalef.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_scalef_ps_128 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vscalefps{{.*}}{%k1} +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vscalefps{{.*}}{%k1} define <4 x float>@test_int_x86_avx512_mask_scalef_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) { %res = call <4 x float> @llvm.x86.avx512.mask.scalef.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) %res1 = call <4 x float> @llvm.x86.avx512.mask.scalef.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1) @@ -2995,12 +2995,2809 @@ define <4 x float>@test_int_x86_avx512_mask_scalef_ps_128(<4 x float> %x0, <4 x declare <8 x float> @llvm.x86.avx512.mask.scalef.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) ; CHECK-LABEL: @test_int_x86_avx512_mask_scalef_ps_256 -; CHECK-NOT: call -; CHECK: kmov -; CHECK: vscalefps{{.*}}{%k1} +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vscalefps{{.*}}{%k1} define <8 x float>@test_int_x86_avx512_mask_scalef_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) { %res = call <8 x float> @llvm.x86.avx512.mask.scalef.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) %res1 = call <8 x float> @llvm.x86.avx512.mask.scalef.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1) %res2 = fadd <8 x float> %res, %res1 ret <8 x float> %res2 -} \ No newline at end of file +} + +declare <2 x double> @llvm.x86.avx512.mask.unpckh.pd.128(<2 x double>, <2 x double>, <2 x double>, i8) + +define <2 x double>@test_int_x86_avx512_mask_unpckh_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_unpckh_pd_128: +; CHECK: vunpckhpd %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: ## xmm2 = xmm2[1],k1[1] +; CHECK-NEXT: vunpckhpd %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0xfd,0x08,0x15,0xc1] +; CHECK-NEXT: ## xmm0 = xmm0[1],xmm1[1] + %res = call <2 x double> @llvm.x86.avx512.mask.unpckh.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) + %res1 = call <2 x double> @llvm.x86.avx512.mask.unpckh.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1) + %res2 = fadd <2 x double> %res, %res1 + ret <2 x double> %res2 +} + +declare <4 x double> @llvm.x86.avx512.mask.unpckh.pd.256(<4 x double>, <4 x double>, <4 x double>, i8) + +define <4 x double>@test_int_x86_avx512_mask_unpckh_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_unpckh_pd_256: +; CHECK: vunpckhpd %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm2[1],k1[1],ymm2[3],k1[3] +; CHECK-NEXT: vunpckhpd %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0xfd,0x28,0x15,0xc1] +; CHECK-NEXT: ## ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] + %res = call <4 x double> @llvm.x86.avx512.mask.unpckh.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) + %res1 = call <4 x double> @llvm.x86.avx512.mask.unpckh.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1) + %res2 = fadd <4 x double> %res, %res1 + ret <4 x double> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.unpckh.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_unpckh_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_unpckh_ps_128: +; CHECK: vunpckhps %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: ## xmm2 = xmm2[2],k1[2],xmm2[3],k1[3] +; CHECK-NEXT: vunpckhps %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7c,0x08,0x15,0xc1] +; CHECK-NEXT: ## xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] + %res = call <4 x float> @llvm.x86.avx512.mask.unpckh.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) + %res1 = call <4 x float> @llvm.x86.avx512.mask.unpckh.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <8 x float> @llvm.x86.avx512.mask.unpckh.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) + +define <8 x float>@test_int_x86_avx512_mask_unpckh_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_unpckh_ps_256: +; CHECK: ## BB#0: +; CHECK: vunpckhps %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm2[2],k1[2],ymm2[3],k1[3],ymm2[6],k1[6],ymm2[7],k1[7] +; CHECK-NEXT: vunpckhps %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7c,0x28,0x15,0xc1] +; CHECK-NEXT: ## ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] + %res = call <8 x float> @llvm.x86.avx512.mask.unpckh.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) + %res1 = call <8 x float> @llvm.x86.avx512.mask.unpckh.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1) + %res2 = fadd <8 x float> %res, %res1 + ret <8 x float> %res2 +} + +declare <2 x double> @llvm.x86.avx512.mask.unpckl.pd.128(<2 x double>, <2 x double>, <2 x double>, i8) + +define <2 x double>@test_int_x86_avx512_mask_unpckl_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_unpckl_pd_128: +; CHECK: vunpcklpd %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: ## xmm2 = xmm2[0],k1[0] +; CHECK-NEXT: vunpcklpd %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0xfd,0x08,0x14,0xc1] +; CHECK-NEXT: ## xmm0 = xmm0[0],xmm1[0] + %res = call <2 x double> @llvm.x86.avx512.mask.unpckl.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) + %res1 = call <2 x double> @llvm.x86.avx512.mask.unpckl.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1) + %res2 = fadd <2 x double> %res, %res1 + ret <2 x double> %res2 +} + +declare <4 x double> @llvm.x86.avx512.mask.unpckl.pd.256(<4 x double>, <4 x double>, <4 x double>, i8) + +define <4 x double>@test_int_x86_avx512_mask_unpckl_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_unpckl_pd_256: +; CHECK: vunpcklpd %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm2[0],k1[0],ymm2[2],k1[2] +; CHECK-NEXT: vunpcklpd %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0xfd,0x28,0x14,0xc1] +; CHECK-NEXT: ## ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] + %res = call <4 x double> @llvm.x86.avx512.mask.unpckl.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) + %res1 = call <4 x double> @llvm.x86.avx512.mask.unpckl.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1) + %res2 = fadd <4 x double> %res, %res1 + ret <4 x double> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.unpckl.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_unpckl_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_unpckl_ps_128: +; CHECK: vunpcklps %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: ## xmm2 = xmm2[0],k1[0],xmm2[1],k1[1] +; CHECK-NEXT: vunpcklps %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7c,0x08,0x14,0xc1] +; CHECK-NEXT: ## xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] + %res = call <4 x float> @llvm.x86.avx512.mask.unpckl.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) + %res1 = call <4 x float> @llvm.x86.avx512.mask.unpckl.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <8 x float> @llvm.x86.avx512.mask.unpckl.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) + +define <8 x float>@test_int_x86_avx512_mask_unpckl_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_unpckl_ps_256: +; CHECK: vunpcklps %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm2[0],k1[0],ymm2[1],k1[1],ymm2[4],k1[4],ymm2[5],k1[5] +; CHECK-NEXT: vunpcklps %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7c,0x28,0x14,0xc1] +; CHECK-NEXT: ## ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] + %res = call <8 x float> @llvm.x86.avx512.mask.unpckl.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) + %res1 = call <8 x float> @llvm.x86.avx512.mask.unpckl.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1) + %res2 = fadd <8 x float> %res, %res1 + ret <8 x float> %res2 +} + +declare <4 x i32> @llvm.x86.avx512.mask.punpckhd.q.128(<4 x i32>, <4 x i32>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_punpckhd_q_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpckhd_q_128: +; CHECK: vpunpckhdq %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: ## xmm2 = xmm2[2],k1[2],xmm2[3],k1[3] +; CHECK-NEXT: vpunpckhdq %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7d,0x08,0x6a,0xc1] +; CHECK-NEXT: ## xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] + %res = call <4 x i32> @llvm.x86.avx512.mask.punpckhd.q.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.punpckhd.q.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1) + %res2 = add <4 x i32> %res, %res1 + ret <4 x i32> %res2 +} + +declare <4 x i32> @llvm.x86.avx512.mask.punpckld.q.128(<4 x i32>, <4 x i32>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_punpckld_q_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpckld_q_128: +; CHECK: vpunpckldq %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: ## xmm2 = xmm2[0],k1[0],xmm2[1],k1[1] +; CHECK-NEXT: vpunpckldq %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0x7d,0x08,0x62,0xc1] +; CHECK-NEXT: ## xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] + %res = call <4 x i32> @llvm.x86.avx512.mask.punpckld.q.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.punpckld.q.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1) + %res2 = add <4 x i32> %res, %res1 + ret <4 x i32> %res2 +} + +declare <8 x i32> @llvm.x86.avx512.mask.punpckhd.q.256(<8 x i32>, <8 x i32>, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_mask_punpckhd_q_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpckhd_q_256: +; CHECK: ## BB#0: +; CHECK: vpunpckhdq %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm2[2],k1[2],ymm2[3],k1[3],ymm2[6],k1[6],ymm2[7],k1[7] +; CHECK-NEXT: vpunpckhdq %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7d,0x28,0x6a,0xc1] +; CHECK-NEXT: ## ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] + %res = call <8 x i32> @llvm.x86.avx512.mask.punpckhd.q.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.punpckhd.q.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1) + %res2 = add <8 x i32> %res, %res1 + ret <8 x i32> %res2 +} + +declare <8 x i32> @llvm.x86.avx512.mask.punpckld.q.256(<8 x i32>, <8 x i32>, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_mask_punpckld_q_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpckld_q_256: +; CHECK: vpunpckldq %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm2[0],k1[0],ymm2[1],k1[1],ymm2[4],k1[4],ymm2[5],k1[5] +; CHECK-NEXT: vpunpckldq %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0x7d,0x28,0x62,0xc1] +; CHECK-NEXT: ## ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] + %res = call <8 x i32> @llvm.x86.avx512.mask.punpckld.q.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.punpckld.q.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1) + %res2 = add <8 x i32> %res, %res1 + ret <8 x i32> %res2 +} + +declare <2 x i64> @llvm.x86.avx512.mask.punpckhqd.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8) + +define <2 x i64>@test_int_x86_avx512_mask_punpckhqd_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpckhqd_q_128: +; CHECK: vpunpckhqdq %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: ## xmm2 = xmm2[1],k1[1] +; CHECK-NEXT: vpunpckhqdq %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0xfd,0x08,0x6d,0xc1] +; CHECK-NEXT: ## xmm0 = xmm0[1],xmm1[1] + %res = call <2 x i64> @llvm.x86.avx512.mask.punpckhqd.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) + %res1 = call <2 x i64> @llvm.x86.avx512.mask.punpckhqd.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1) + %res2 = add <2 x i64> %res, %res1 + ret <2 x i64> %res2 +} + +declare <2 x i64> @llvm.x86.avx512.mask.punpcklqd.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8) + +define <2 x i64>@test_int_x86_avx512_mask_punpcklqd_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpcklqd_q_128: +; CHECK: vpunpcklqdq %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: ## xmm2 = xmm2[0],k1[0] +; CHECK-NEXT: vpunpcklqdq %xmm1, %xmm0, %xmm0 ## encoding: [0x62,0xf1,0xfd,0x08,0x6c,0xc1] +; CHECK-NEXT: ## xmm0 = xmm0[0],xmm1[0] + %res = call <2 x i64> @llvm.x86.avx512.mask.punpcklqd.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) + %res1 = call <2 x i64> @llvm.x86.avx512.mask.punpcklqd.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1) + %res2 = add <2 x i64> %res, %res1 + ret <2 x i64> %res2 +} + +declare <4 x i64> @llvm.x86.avx512.mask.punpcklqd.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8) + +define <4 x i64>@test_int_x86_avx512_mask_punpcklqd_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpcklqd_q_256: +; CHECK: vpunpcklqdq %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm2[0],k1[0],ymm2[2],k1[2] +; CHECK-NEXT: vpunpcklqdq %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0xfd,0x28,0x6c,0xc1] +; CHECK-NEXT: ## ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] + %res = call <4 x i64> @llvm.x86.avx512.mask.punpcklqd.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) + %res1 = call <4 x i64> @llvm.x86.avx512.mask.punpcklqd.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1) + %res2 = add <4 x i64> %res, %res1 + ret <4 x i64> %res2 +} + +declare <4 x i64> @llvm.x86.avx512.mask.punpckhqd.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8) + +define <4 x i64>@test_int_x86_avx512_mask_punpckhqd_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_punpckhqd_q_256: +; CHECK: vpunpckhqdq %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm2[1],k1[1],ymm2[3],k1[3] +; CHECK-NEXT: vpunpckhqdq %ymm1, %ymm0, %ymm0 ## encoding: [0x62,0xf1,0xfd,0x28,0x6d,0xc1] +; CHECK-NEXT: ## ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] + %res = call <4 x i64> @llvm.x86.avx512.mask.punpckhqd.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) + %res1 = call <4 x i64> @llvm.x86.avx512.mask.punpckhqd.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1) + %res2 = add <4 x i64> %res, %res1 + ret <4 x i64> %res2 +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmov_qb_128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qb_128: +; CHECK: vpmovqb %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovqb %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovqb %xmm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.qb.mem.128(i8* %ptr, <2 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmov_qb_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qb_mem_128: +; CHECK: vpmovqb %xmm0, (%rdi) +; CHECK: vpmovqb %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmov.qb.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmov.qb.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmovs_qb_128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qb_128: +; CHECK: vpmovsqb %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovsqb %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovsqb %xmm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.qb.mem.128(i8* %ptr, <2 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovs_qb_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qb_mem_128: +; CHECK: vpmovsqb %xmm0, (%rdi) +; CHECK: vpmovsqb %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovs.qb.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovs.qb.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmovus_qb_128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qb_128: +; CHECK: vpmovusqb %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovusqb %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovusqb %xmm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.qb.mem.128(i8* %ptr, <2 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovus_qb_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qb_mem_128: +; CHECK: vpmovusqb %xmm0, (%rdi) +; CHECK: vpmovusqb %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovus.qb.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovus.qb.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmov.qb.256(<4 x i64>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmov_qb_256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qb_256: +; CHECK: vpmovqb %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovqb %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovqb %ymm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.256(<4 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.qb.mem.256(i8* %ptr, <4 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmov_qb_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qb_mem_256: +; CHECK: vpmovqb %ymm0, (%rdi) +; CHECK: vpmovqb %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmov.qb.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmov.qb.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256(<4 x i64>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmovs_qb_256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qb_256: +; CHECK: vpmovsqb %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovsqb %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovsqb %ymm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256(<4 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.qb.mem.256(i8* %ptr, <4 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovs_qb_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qb_mem_256: +; CHECK: vpmovsqb %ymm0, (%rdi) +; CHECK: vpmovsqb %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovs.qb.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovs.qb.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.256(<4 x i64>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmovus_qb_256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qb_256: +; CHECK: vpmovusqb %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovusqb %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovusqb %ymm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.256(<4 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.qb.mem.256(i8* %ptr, <4 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovus_qb_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qb_mem_256: +; CHECK: vpmovusqb %ymm0, (%rdi) +; CHECK: vpmovusqb %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovus.qb.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovus.qb.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2) + ret void +} + +declare <8 x i16> @llvm.x86.avx512.mask.pmov.qw.128(<2 x i64>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_pmov_qw_128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qw_128: +; CHECK: vpmovqw %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovqw %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovqw %xmm0, %xmm0 + %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 -1) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.128(<2 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2) + %res3 = add <8 x i16> %res0, %res1 + %res4 = add <8 x i16> %res3, %res2 + ret <8 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.qw.mem.128(i8* %ptr, <2 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmov_qw_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qw_mem_128: +; CHECK: vpmovqw %xmm0, (%rdi) +; CHECK: vpmovqw %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmov.qw.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmov.qw.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2) + ret void +} + +declare <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.128(<2 x i64>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_pmovs_qw_128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qw_128: +; CHECK: vpmovsqw %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovsqw %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovsqw %xmm0, %xmm0 + %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 -1) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.128(<2 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2) + %res3 = add <8 x i16> %res0, %res1 + %res4 = add <8 x i16> %res3, %res2 + ret <8 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.qw.mem.128(i8* %ptr, <2 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovs_qw_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qw_mem_128: +; CHECK: vpmovsqw %xmm0, (%rdi) +; CHECK: vpmovsqw %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovs.qw.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovs.qw.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2) + ret void +} + +declare <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.128(<2 x i64>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_pmovus_qw_128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qw_128: +; CHECK: vpmovusqw %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovusqw %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovusqw %xmm0, %xmm0 + %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 -1) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.128(<2 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2) + %res3 = add <8 x i16> %res0, %res1 + %res4 = add <8 x i16> %res3, %res2 + ret <8 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.qw.mem.128(i8* %ptr, <2 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovus_qw_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qw_mem_128: +; CHECK: vpmovusqw %xmm0, (%rdi) +; CHECK: vpmovusqw %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovus.qw.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovus.qw.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2) + ret void +} + +declare <8 x i16> @llvm.x86.avx512.mask.pmov.qw.256(<4 x i64>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_pmov_qw_256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qw_256: +; CHECK: vpmovqw %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovqw %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovqw %ymm0, %xmm0 + %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 -1) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.256(<4 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2) + %res3 = add <8 x i16> %res0, %res1 + %res4 = add <8 x i16> %res3, %res2 + ret <8 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.qw.mem.256(i8* %ptr, <4 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmov_qw_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qw_mem_256: +; CHECK: vpmovqw %ymm0, (%rdi) +; CHECK: vpmovqw %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmov.qw.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmov.qw.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2) + ret void +} + +declare <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.256(<4 x i64>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_pmovs_qw_256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qw_256: +; CHECK: vpmovsqw %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovsqw %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovsqw %ymm0, %xmm0 + %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 -1) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.256(<4 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2) + %res3 = add <8 x i16> %res0, %res1 + %res4 = add <8 x i16> %res3, %res2 + ret <8 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.qw.mem.256(i8* %ptr, <4 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovs_qw_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qw_mem_256: +; CHECK: vpmovsqw %ymm0, (%rdi) +; CHECK: vpmovsqw %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovs.qw.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovs.qw.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2) + ret void +} + +declare <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.256(<4 x i64>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_pmovus_qw_256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qw_256: +; CHECK: vpmovusqw %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovusqw %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovusqw %ymm0, %xmm0 + %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 -1) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 %x2) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.256(<4 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2) + %res3 = add <8 x i16> %res0, %res1 + %res4 = add <8 x i16> %res3, %res2 + ret <8 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.qw.mem.256(i8* %ptr, <4 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovus_qw_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qw_mem_256: +; CHECK: vpmovusqw %ymm0, (%rdi) +; CHECK: vpmovusqw %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovus.qw.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovus.qw.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2) + ret void +} + +declare <4 x i32> @llvm.x86.avx512.mask.pmov.qd.128(<2 x i64>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_pmov_qd_128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qd_128: +; CHECK: vpmovqd %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovqd %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovqd %xmm0, %xmm0 + %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 -1) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2) + %res2 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.128(<2 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2) + %res3 = add <4 x i32> %res0, %res1 + %res4 = add <4 x i32> %res3, %res2 + ret <4 x i32> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.qd.mem.128(i8* %ptr, <2 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmov_qd_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qd_mem_128: +; CHECK: vpmovqd %xmm0, (%rdi) +; CHECK: vpmovqd %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmov.qd.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmov.qd.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2) + ret void +} + +declare <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.128(<2 x i64>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_pmovs_qd_128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qd_128: +; CHECK: vpmovsqd %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovsqd %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovsqd %xmm0, %xmm0 + %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 -1) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2) + %res2 = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.128(<2 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2) + %res3 = add <4 x i32> %res0, %res1 + %res4 = add <4 x i32> %res3, %res2 + ret <4 x i32> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.qd.mem.128(i8* %ptr, <2 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovs_qd_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qd_mem_128: +; CHECK: vpmovsqd %xmm0, (%rdi) +; CHECK: vpmovsqd %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovs.qd.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovs.qd.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2) + ret void +} + +declare <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.128(<2 x i64>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_pmovus_qd_128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qd_128: +; CHECK: vpmovusqd %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovusqd %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovusqd %xmm0, %xmm0 + %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 -1) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2) + %res2 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.128(<2 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2) + %res3 = add <4 x i32> %res0, %res1 + %res4 = add <4 x i32> %res3, %res2 + ret <4 x i32> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.qd.mem.128(i8* %ptr, <2 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovus_qd_mem_128(i8* %ptr, <2 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qd_mem_128: +; CHECK: vpmovusqd %xmm0, (%rdi) +; CHECK: vpmovusqd %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovus.qd.mem.128(i8* %ptr, <2 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovus.qd.mem.128(i8* %ptr, <2 x i64> %x1, i8 %x2) + ret void +} + +declare <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_pmov_qd_256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qd_256: +; CHECK: vpmovqd %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovqd %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovqd %ymm0, %xmm0 + %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 -1) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2) + %res2 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2) + %res3 = add <4 x i32> %res0, %res1 + %res4 = add <4 x i32> %res3, %res2 + ret <4 x i32> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.qd.mem.256(i8* %ptr, <4 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmov_qd_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qd_mem_256: +; CHECK: vpmovqd %ymm0, (%rdi) +; CHECK: vpmovqd %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmov.qd.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmov.qd.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2) + ret void +} + +declare <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.256(<4 x i64>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_pmovs_qd_256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qd_256: +; CHECK: vpmovsqd %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovsqd %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovsqd %ymm0, %xmm0 + %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 -1) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2) + %res2 = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.256(<4 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2) + %res3 = add <4 x i32> %res0, %res1 + %res4 = add <4 x i32> %res3, %res2 + ret <4 x i32> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.qd.mem.256(i8* %ptr, <4 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovs_qd_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qd_mem_256: +; CHECK: vpmovsqd %ymm0, (%rdi) +; CHECK: vpmovsqd %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovs.qd.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovs.qd.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2) + ret void +} + +declare <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.256(<4 x i64>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_pmovus_qd_256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qd_256: +; CHECK: vpmovusqd %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovusqd %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovusqd %ymm0, %xmm0 + %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 -1) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2) + %res2 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.256(<4 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2) + %res3 = add <4 x i32> %res0, %res1 + %res4 = add <4 x i32> %res3, %res2 + ret <4 x i32> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.qd.mem.256(i8* %ptr, <4 x i64>, i8) + +define void @test_int_x86_avx512_mask_pmovus_qd_mem_256(i8* %ptr, <4 x i64> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qd_mem_256: +; CHECK: vpmovusqd %ymm0, (%rdi) +; CHECK: vpmovusqd %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovus.qd.mem.256(i8* %ptr, <4 x i64> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovus.qd.mem.256(i8* %ptr, <4 x i64> %x1, i8 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmov_db_128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_db_128: +; CHECK: vpmovdb %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovdb %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovdb %xmm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.db.mem.128(i8* %ptr, <4 x i32>, i8) + +define void @test_int_x86_avx512_mask_pmov_db_mem_128(i8* %ptr, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_db_mem_128: +; CHECK: vpmovdb %xmm0, (%rdi) +; CHECK: vpmovdb %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmov.db.mem.128(i8* %ptr, <4 x i32> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmov.db.mem.128(i8* %ptr, <4 x i32> %x1, i8 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmovs_db_128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_db_128: +; CHECK: vpmovsdb %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovsdb %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovsdb %xmm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.db.mem.128(i8* %ptr, <4 x i32>, i8) + +define void @test_int_x86_avx512_mask_pmovs_db_mem_128(i8* %ptr, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_db_mem_128: +; CHECK: vpmovsdb %xmm0, (%rdi) +; CHECK: vpmovsdb %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovs.db.mem.128(i8* %ptr, <4 x i32> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovs.db.mem.128(i8* %ptr, <4 x i32> %x1, i8 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmovus_db_128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_db_128: +; CHECK: vpmovusdb %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovusdb %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovusdb %xmm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.db.mem.128(i8* %ptr, <4 x i32>, i8) + +define void @test_int_x86_avx512_mask_pmovus_db_mem_128(i8* %ptr, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_db_mem_128: +; CHECK: vpmovusdb %xmm0, (%rdi) +; CHECK: vpmovusdb %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovus.db.mem.128(i8* %ptr, <4 x i32> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovus.db.mem.128(i8* %ptr, <4 x i32> %x1, i8 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmov.db.256(<8 x i32>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmov_db_256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_db_256: +; CHECK: vpmovdb %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovdb %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovdb %ymm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.256(<8 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.db.mem.256(i8* %ptr, <8 x i32>, i8) + +define void @test_int_x86_avx512_mask_pmov_db_mem_256(i8* %ptr, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_db_mem_256: +; CHECK: vpmovdb %ymm0, (%rdi) +; CHECK: vpmovdb %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmov.db.mem.256(i8* %ptr, <8 x i32> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmov.db.mem.256(i8* %ptr, <8 x i32> %x1, i8 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmovs.db.256(<8 x i32>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmovs_db_256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_db_256: +; CHECK: vpmovsdb %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovsdb %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovsdb %ymm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.256(<8 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.db.mem.256(i8* %ptr, <8 x i32>, i8) + +define void @test_int_x86_avx512_mask_pmovs_db_mem_256(i8* %ptr, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_db_mem_256: +; CHECK: vpmovsdb %ymm0, (%rdi) +; CHECK: vpmovsdb %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovs.db.mem.256(i8* %ptr, <8 x i32> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovs.db.mem.256(i8* %ptr, <8 x i32> %x1, i8 %x2) + ret void +} + +declare <16 x i8> @llvm.x86.avx512.mask.pmovus.db.256(<8 x i32>, <16 x i8>, i8) + +define <16 x i8>@test_int_x86_avx512_mask_pmovus_db_256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_db_256: +; CHECK: vpmovusdb %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovusdb %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovusdb %ymm0, %xmm0 + %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 -1) + %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2) + %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.256(<8 x i32> %x0, <16 x i8> zeroinitializer, i8 %x2) + %res3 = add <16 x i8> %res0, %res1 + %res4 = add <16 x i8> %res3, %res2 + ret <16 x i8> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.db.mem.256(i8* %ptr, <8 x i32>, i8) + +define void @test_int_x86_avx512_mask_pmovus_db_mem_256(i8* %ptr, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_db_mem_256: +; CHECK: vpmovusdb %ymm0, (%rdi) +; CHECK: vpmovusdb %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovus.db.mem.256(i8* %ptr, <8 x i32> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovus.db.mem.256(i8* %ptr, <8 x i32> %x1, i8 %x2) + ret void +} + +declare <8 x i16> @llvm.x86.avx512.mask.pmov.dw.128(<4 x i32>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_pmov_dw_128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_dw_128: +; CHECK: vpmovdw %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovdw %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovdw %xmm0, %xmm0 + %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 -1) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.128(<4 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2) + %res3 = add <8 x i16> %res0, %res1 + %res4 = add <8 x i16> %res3, %res2 + ret <8 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.dw.mem.128(i8* %ptr, <4 x i32>, i8) + +define void @test_int_x86_avx512_mask_pmov_dw_mem_128(i8* %ptr, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_dw_mem_128: +; CHECK: vpmovdw %xmm0, (%rdi) +; CHECK: vpmovdw %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmov.dw.mem.128(i8* %ptr, <4 x i32> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmov.dw.mem.128(i8* %ptr, <4 x i32> %x1, i8 %x2) + ret void +} + +declare <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.128(<4 x i32>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_pmovs_dw_128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_dw_128: +; CHECK: vpmovsdw %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovsdw %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovsdw %xmm0, %xmm0 + %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 -1) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.128(<4 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2) + %res3 = add <8 x i16> %res0, %res1 + %res4 = add <8 x i16> %res3, %res2 + ret <8 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.dw.mem.128(i8* %ptr, <4 x i32>, i8) + +define void @test_int_x86_avx512_mask_pmovs_dw_mem_128(i8* %ptr, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_dw_mem_128: +; CHECK: vpmovsdw %xmm0, (%rdi) +; CHECK: vpmovsdw %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovs.dw.mem.128(i8* %ptr, <4 x i32> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovs.dw.mem.128(i8* %ptr, <4 x i32> %x1, i8 %x2) + ret void +} + +declare <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.128(<4 x i32>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_pmovus_dw_128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_dw_128: +; CHECK: vpmovusdw %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovusdw %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovusdw %xmm0, %xmm0 + %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 -1) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.128(<4 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2) + %res3 = add <8 x i16> %res0, %res1 + %res4 = add <8 x i16> %res3, %res2 + ret <8 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.dw.mem.128(i8* %ptr, <4 x i32>, i8) + +define void @test_int_x86_avx512_mask_pmovus_dw_mem_128(i8* %ptr, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_dw_mem_128: +; CHECK: vpmovusdw %xmm0, (%rdi) +; CHECK: vpmovusdw %xmm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovus.dw.mem.128(i8* %ptr, <4 x i32> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovus.dw.mem.128(i8* %ptr, <4 x i32> %x1, i8 %x2) + ret void +} + +declare <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_pmov_dw_256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_dw_256: +; CHECK: vpmovdw %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovdw %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovdw %ymm0, %xmm0 + %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 -1) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2) + %res3 = add <8 x i16> %res0, %res1 + %res4 = add <8 x i16> %res3, %res2 + ret <8 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmov.dw.mem.256(i8* %ptr, <8 x i32>, i8) + +define void @test_int_x86_avx512_mask_pmov_dw_mem_256(i8* %ptr, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmov_dw_mem_256: +; CHECK: vpmovdw %ymm0, (%rdi) +; CHECK: vpmovdw %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmov.dw.mem.256(i8* %ptr, <8 x i32> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmov.dw.mem.256(i8* %ptr, <8 x i32> %x1, i8 %x2) + ret void +} + +declare <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_pmovs_dw_256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_dw_256: +; CHECK: vpmovsdw %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovsdw %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovsdw %ymm0, %xmm0 + %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 -1) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2) + %res3 = add <8 x i16> %res0, %res1 + %res4 = add <8 x i16> %res3, %res2 + ret <8 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovs.dw.mem.256(i8* %ptr, <8 x i32>, i8) + +define void @test_int_x86_avx512_mask_pmovs_dw_mem_256(i8* %ptr, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_dw_mem_256: +; CHECK: vpmovsdw %ymm0, (%rdi) +; CHECK: vpmovsdw %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovs.dw.mem.256(i8* %ptr, <8 x i32> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovs.dw.mem.256(i8* %ptr, <8 x i32> %x1, i8 %x2) + ret void +} + +declare <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32>, <8 x i16>, i8) + +define <8 x i16>@test_int_x86_avx512_mask_pmovus_dw_256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_dw_256: +; CHECK: vpmovusdw %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vpmovusdw %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpmovusdw %ymm0, %xmm0 + %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 -1) + %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 %x2) + %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32> %x0, <8 x i16> zeroinitializer, i8 %x2) + %res3 = add <8 x i16> %res0, %res1 + %res4 = add <8 x i16> %res3, %res2 + ret <8 x i16> %res4 +} + +declare void @llvm.x86.avx512.mask.pmovus.dw.mem.256(i8* %ptr, <8 x i32>, i8) + +define void @test_int_x86_avx512_mask_pmovus_dw_mem_256(i8* %ptr, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_dw_mem_256: +; CHECK: vpmovusdw %ymm0, (%rdi) +; CHECK: vpmovusdw %ymm0, (%rdi) {%k1} + call void @llvm.x86.avx512.mask.pmovus.dw.mem.256(i8* %ptr, <8 x i32> %x1, i8 -1) + call void @llvm.x86.avx512.mask.pmovus.dw.mem.256(i8* %ptr, <8 x i32> %x1, i8 %x2) + ret void +} + +declare <2 x double> @llvm.x86.avx512.mask.cvtdq2pd.128(<4 x i32>, <2 x double>, i8) + +define <2 x double>@test_int_x86_avx512_mask_cvt_dq2pd_128(<4 x i32> %x0, <2 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_dq2pd_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtdq2pd %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtdq2pd %xmm0, %xmm0 +; CHECK-NEXT: vaddpd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.cvtdq2pd.128(<4 x i32> %x0, <2 x double> %x1, i8 %x2) + %res1 = call <2 x double> @llvm.x86.avx512.mask.cvtdq2pd.128(<4 x i32> %x0, <2 x double> %x1, i8 -1) + %res2 = fadd <2 x double> %res, %res1 + ret <2 x double> %res2 +} + +declare <4 x double> @llvm.x86.avx512.mask.cvtdq2pd.256(<4 x i32>, <4 x double>, i8) + +define <4 x double>@test_int_x86_avx512_mask_cvt_dq2pd_256(<4 x i32> %x0, <4 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_dq2pd_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtdq2pd %xmm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtdq2pd %xmm0, %ymm0 +; CHECK-NEXT: vaddpd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.avx512.mask.cvtdq2pd.256(<4 x i32> %x0, <4 x double> %x1, i8 %x2) + %res1 = call <4 x double> @llvm.x86.avx512.mask.cvtdq2pd.256(<4 x i32> %x0, <4 x double> %x1, i8 -1) + %res2 = fadd <4 x double> %res, %res1 + ret <4 x double> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.cvtdq2ps.128(<4 x i32>, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_cvt_dq2ps_128(<4 x i32> %x0, <4 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_dq2ps_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtdq2ps %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtdq2ps %xmm0, %xmm0 +; CHECK-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.cvtdq2ps.128(<4 x i32> %x0, <4 x float> %x1, i8 %x2) + %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtdq2ps.128(<4 x i32> %x0, <4 x float> %x1, i8 -1) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <8 x float> @llvm.x86.avx512.mask.cvtdq2ps.256(<8 x i32>, <8 x float>, i8) + +define <8 x float>@test_int_x86_avx512_mask_cvt_dq2ps_256(<8 x i32> %x0, <8 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_dq2ps_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtdq2ps %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtdq2ps %ymm0, %ymm0 +; CHECK-NEXT: vaddps %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.avx512.mask.cvtdq2ps.256(<8 x i32> %x0, <8 x float> %x1, i8 %x2) + %res1 = call <8 x float> @llvm.x86.avx512.mask.cvtdq2ps.256(<8 x i32> %x0, <8 x float> %x1, i8 -1) + %res2 = fadd <8 x float> %res, %res1 + ret <8 x float> %res2 +} + +declare <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.128(<2 x double>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_cvt_pd2dq_128(<2 x double> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2dq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtpd2dq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtpd2dq %xmm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.128(<2 x double> %x0, <4 x i32> %x1, i8 %x2) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.128(<2 x double> %x0, <4 x i32> %x1, i8 -1) + %res2 = add <4 x i32> %res, %res1 + ret <4 x i32> %res2 +} + +declare <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.256(<4 x double>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_cvt_pd2dq_256(<4 x double> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2dq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtpd2dq %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtpd2dq %ymm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.256(<4 x double> %x0, <4 x i32> %x1, i8 %x2) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2dq.256(<4 x double> %x0, <4 x i32> %x1, i8 -1) + %res2 = add <4 x i32> %res, %res1 + ret <4 x i32> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.cvtpd2ps.256(<4 x double>, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_cvt_pd2ps_256(<4 x double> %x0, <4 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2ps_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtpd2ps %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtpd2ps %ymm0, %xmm0 +; CHECK-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.cvtpd2ps.256(<4 x double> %x0, <4 x float> %x1, i8 %x2) + %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtpd2ps.256(<4 x double> %x0, <4 x float> %x1, i8 -1) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.cvtpd2ps(<2 x double>, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_cvt_pd2ps(<2 x double> %x0, <4 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2ps: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtpd2ps %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtpd2ps %xmm0, %xmm0 +; CHECK-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.cvtpd2ps(<2 x double> %x0, <4 x float> %x1, i8 %x2) + %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtpd2ps(<2 x double> %x0, <4 x float> %x1, i8 -1) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.128(<2 x double>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_cvt_pd2udq_128(<2 x double> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2udq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtpd2udq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtpd2udq %xmm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.128(<2 x double> %x0, <4 x i32> %x1, i8 %x2) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.128(<2 x double> %x0, <4 x i32> %x1, i8 -1) + %res2 = add <4 x i32> %res, %res1 + ret <4 x i32> %res2 +} + +declare <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.256(<4 x double>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_cvt_pd2udq_256(<4 x double> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2udq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtpd2udq %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtpd2udq %ymm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.256(<4 x double> %x0, <4 x i32> %x1, i8 %x2) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvtpd2udq.256(<4 x double> %x0, <4 x i32> %x1, i8 -1) + %res2 = add <4 x i32> %res, %res1 + ret <4 x i32> %res2 +} + +declare <4 x i32> @llvm.x86.avx512.mask.cvtps2dq.128(<4 x float>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_cvt_ps2dq_128(<4 x float> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2dq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtps2dq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtps2dq %xmm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.cvtps2dq.128(<4 x float> %x0, <4 x i32> %x1, i8 %x2) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvtps2dq.128(<4 x float> %x0, <4 x i32> %x1, i8 -1) + %res2 = add <4 x i32> %res, %res1 + ret <4 x i32> %res2 +} + +declare <8 x i32> @llvm.x86.avx512.mask.cvtps2dq.256(<8 x float>, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_mask_cvt_ps2dq_256(<8 x float> %x0, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2dq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtps2dq %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtps2dq %ymm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.mask.cvtps2dq.256(<8 x float> %x0, <8 x i32> %x1, i8 %x2) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvtps2dq.256(<8 x float> %x0, <8 x i32> %x1, i8 -1) + %res2 = add <8 x i32> %res, %res1 + ret <8 x i32> %res2 +} + +declare <2 x double> @llvm.x86.avx512.mask.cvtps2pd.128(<4 x float>, <2 x double>, i8) + +define <2 x double>@test_int_x86_avx512_mask_cvt_ps2pd_128(<4 x float> %x0, <2 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2pd_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtps2pd %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtps2pd %xmm0, %xmm0 +; CHECK-NEXT: vaddpd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.cvtps2pd.128(<4 x float> %x0, <2 x double> %x1, i8 %x2) + %res1 = call <2 x double> @llvm.x86.avx512.mask.cvtps2pd.128(<4 x float> %x0, <2 x double> %x1, i8 -1) + %res2 = fadd <2 x double> %res, %res1 + ret <2 x double> %res2 +} + +declare <4 x double> @llvm.x86.avx512.mask.cvtps2pd.256(<4 x float>, <4 x double>, i8) + +define <4 x double>@test_int_x86_avx512_mask_cvt_ps2pd_256(<4 x float> %x0, <4 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2pd_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtps2pd %xmm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtps2pd %xmm0, %ymm0 +; CHECK-NEXT: vaddpd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.avx512.mask.cvtps2pd.256(<4 x float> %x0, <4 x double> %x1, i8 %x2) + %res1 = call <4 x double> @llvm.x86.avx512.mask.cvtps2pd.256(<4 x float> %x0, <4 x double> %x1, i8 -1) + %res2 = fadd <4 x double> %res, %res1 + ret <4 x double> %res2 +} + +declare <4 x i32> @llvm.x86.avx512.mask.cvtps2udq.128(<4 x float>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_cvt_ps2udq_128(<4 x float> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2udq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtps2udq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtps2udq %xmm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.cvtps2udq.128(<4 x float> %x0, <4 x i32> %x1, i8 %x2) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvtps2udq.128(<4 x float> %x0, <4 x i32> %x1, i8 -1) + %res2 = add <4 x i32> %res, %res1 + ret <4 x i32> %res2 +} + +declare <8 x i32> @llvm.x86.avx512.mask.cvtps2udq.256(<8 x float>, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_mask_cvt_ps2udq_256(<8 x float> %x0, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2udq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtps2udq %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtps2udq %ymm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.mask.cvtps2udq.256(<8 x float> %x0, <8 x i32> %x1, i8 %x2) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvtps2udq.256(<8 x float> %x0, <8 x i32> %x1, i8 -1) + %res2 = add <8 x i32> %res, %res1 + ret <8 x i32> %res2 +} + +declare <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.128(<2 x double>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_cvtt_pd2dq_128(<2 x double> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2dq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvttpd2dq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvttpd2dq %xmm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.128(<2 x double> %x0, <4 x i32> %x1, i8 %x2) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.128(<2 x double> %x0, <4 x i32> %x1, i8 -1) + %res2 = add <4 x i32> %res, %res1 + ret <4 x i32> %res2 +} + +declare <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.256(<4 x double>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_cvtt_pd2dq_256(<4 x double> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2dq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvttpd2dq %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vcvttpd2dq %ymm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.256(<4 x double> %x0, <4 x i32> %x1, i8 %x2) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2dq.256(<4 x double> %x0, <4 x i32> %x1, i8 -1) + %res2 = add <4 x i32> %res, %res1 + ret <4 x i32> %res2 +} + +declare <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.128(<2 x double>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_cvtt_pd2udq_128(<2 x double> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2udq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvttpd2udq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvttpd2udq %xmm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.128(<2 x double> %x0, <4 x i32> %x1, i8 %x2) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.128(<2 x double> %x0, <4 x i32> %x1, i8 -1) + %res2 = add <4 x i32> %res, %res1 + ret <4 x i32> %res2 +} + +declare <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.256(<4 x double>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_cvtt_pd2udq_256(<4 x double> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2udq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvttpd2udq %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vcvttpd2udq %ymm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.256(<4 x double> %x0, <4 x i32> %x1, i8 %x2) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvttpd2udq.256(<4 x double> %x0, <4 x i32> %x1, i8 -1) + %res2 = add <4 x i32> %res, %res1 + ret <4 x i32> %res2 +} + +declare <4 x i32> @llvm.x86.avx512.mask.cvttps2dq.128(<4 x float>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_cvtt_ps2dq_128(<4 x float> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2dq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvttps2dq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvttps2dq %xmm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.cvttps2dq.128(<4 x float> %x0, <4 x i32> %x1, i8 %x2) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvttps2dq.128(<4 x float> %x0, <4 x i32> %x1, i8 -1) + %res2 = add <4 x i32> %res, %res1 + ret <4 x i32> %res2 +} + +declare <8 x i32> @llvm.x86.avx512.mask.cvttps2dq.256(<8 x float>, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_mask_cvtt_ps2dq_256(<8 x float> %x0, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2dq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvttps2dq %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vcvttps2dq %ymm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.mask.cvttps2dq.256(<8 x float> %x0, <8 x i32> %x1, i8 %x2) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvttps2dq.256(<8 x float> %x0, <8 x i32> %x1, i8 -1) + %res2 = add <8 x i32> %res, %res1 + ret <8 x i32> %res2 +} + +declare <4 x i32> @llvm.x86.avx512.mask.cvttps2udq.128(<4 x float>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_cvtt_ps2udq_128(<4 x float> %x0, <4 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2udq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvttps2udq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvttps2udq %xmm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.cvttps2udq.128(<4 x float> %x0, <4 x i32> %x1, i8 %x2) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.cvttps2udq.128(<4 x float> %x0, <4 x i32> %x1, i8 -1) + %res2 = add <4 x i32> %res, %res1 + ret <4 x i32> %res2 +} + +declare <8 x i32> @llvm.x86.avx512.mask.cvttps2udq.256(<8 x float>, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_mask_cvtt_ps2udq_256(<8 x float> %x0, <8 x i32> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2udq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvttps2udq %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vcvttps2udq %ymm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.mask.cvttps2udq.256(<8 x float> %x0, <8 x i32> %x1, i8 %x2) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvttps2udq.256(<8 x float> %x0, <8 x i32> %x1, i8 -1) + %res2 = add <8 x i32> %res, %res1 + ret <8 x i32> %res2 +} + +declare <2 x double> @llvm.x86.avx512.mask.cvtudq2pd.128(<4 x i32>, <2 x double>, i8) + +define <2 x double>@test_int_x86_avx512_mask_cvt_udq2pd_128(<4 x i32> %x0, <2 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_udq2pd_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtudq2pd %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtudq2pd %xmm0, %xmm0 +; CHECK-NEXT: vaddpd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.cvtudq2pd.128(<4 x i32> %x0, <2 x double> %x1, i8 %x2) + %res1 = call <2 x double> @llvm.x86.avx512.mask.cvtudq2pd.128(<4 x i32> %x0, <2 x double> %x1, i8 -1) + %res2 = fadd <2 x double> %res, %res1 + ret <2 x double> %res2 +} + +declare <4 x double> @llvm.x86.avx512.mask.cvtudq2pd.256(<4 x i32>, <4 x double>, i8) + +define <4 x double>@test_int_x86_avx512_mask_cvt_udq2pd_256(<4 x i32> %x0, <4 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_udq2pd_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtudq2pd %xmm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtudq2pd %xmm0, %ymm0 +; CHECK-NEXT: vaddpd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.avx512.mask.cvtudq2pd.256(<4 x i32> %x0, <4 x double> %x1, i8 %x2) + %res1 = call <4 x double> @llvm.x86.avx512.mask.cvtudq2pd.256(<4 x i32> %x0, <4 x double> %x1, i8 -1) + %res2 = fadd <4 x double> %res, %res1 + ret <4 x double> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.cvtudq2ps.128(<4 x i32>, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_cvt_udq2ps_128(<4 x i32> %x0, <4 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_udq2ps_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtudq2ps %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vcvtudq2ps %xmm0, %xmm0 +; CHECK-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.cvtudq2ps.128(<4 x i32> %x0, <4 x float> %x1, i8 %x2) + %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtudq2ps.128(<4 x i32> %x0, <4 x float> %x1, i8 -1) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <8 x float> @llvm.x86.avx512.mask.cvtudq2ps.256(<8 x i32>, <8 x float>, i8) + +define <8 x float>@test_int_x86_avx512_mask_cvt_udq2ps_256(<8 x i32> %x0, <8 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_cvt_udq2ps_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vcvtudq2ps %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vcvtudq2ps %ymm0, %ymm0 +; CHECK-NEXT: vaddps %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.avx512.mask.cvtudq2ps.256(<8 x i32> %x0, <8 x float> %x1, i8 %x2) + %res1 = call <8 x float> @llvm.x86.avx512.mask.cvtudq2ps.256(<8 x i32> %x0, <8 x float> %x1, i8 -1) + %res2 = fadd <8 x float> %res, %res1 + ret <8 x float> %res2 +} + +declare <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128(<2 x double>, i32, <2 x double>, i8) +; CHECK-LABEL: @test_int_x86_avx512_mask_rndscale_pd_128 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vrndscalepd {{.*}}{%k1} +; CHECK: vrndscalepd +define <2 x double>@test_int_x86_avx512_mask_rndscale_pd_128(<2 x double> %x0, <2 x double> %x2, i8 %x3) { + %res = call <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128(<2 x double> %x0, i32 4, <2 x double> %x2, i8 %x3) + %res1 = call <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128(<2 x double> %x0, i32 88, <2 x double> %x2, i8 -1) + %res2 = fadd <2 x double> %res, %res1 + ret <2 x double> %res2 +} + +declare <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256(<4 x double>, i32, <4 x double>, i8) +; CHECK-LABEL: @test_int_x86_avx512_mask_rndscale_pd_256 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vrndscalepd {{.*}}{%k1} +; CHECK: vrndscalepd +define <4 x double>@test_int_x86_avx512_mask_rndscale_pd_256(<4 x double> %x0, <4 x double> %x2, i8 %x3) { + %res = call <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256(<4 x double> %x0, i32 4, <4 x double> %x2, i8 %x3) + %res1 = call <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256(<4 x double> %x0, i32 88, <4 x double> %x2, i8 -1) + %res2 = fadd <4 x double> %res, %res1 + ret <4 x double> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128(<4 x float>, i32, <4 x float>, i8) +; CHECK-LABEL: @test_int_x86_avx512_mask_rndscale_ps_128 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vrndscaleps {{.*}}{%k1} +; CHECK: vrndscaleps +define <4 x float>@test_int_x86_avx512_mask_rndscale_ps_128(<4 x float> %x0, <4 x float> %x2, i8 %x3) { + %res = call <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128(<4 x float> %x0, i32 88, <4 x float> %x2, i8 %x3) + %res1 = call <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128(<4 x float> %x0, i32 4, <4 x float> %x2, i8 -1) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256(<8 x float>, i32, <8 x float>, i8) + +; CHECK-LABEL: @test_int_x86_avx512_mask_rndscale_ps_256 +; CHECK-NOT: call +; CHECK: kmov +; CHECK: vrndscaleps {{.*}}{%k1} +; CHECK: vrndscaleps +define <8 x float>@test_int_x86_avx512_mask_rndscale_ps_256(<8 x float> %x0, <8 x float> %x2, i8 %x3) { + %res = call <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256(<8 x float> %x0, i32 5, <8 x float> %x2, i8 %x3) + %res1 = call <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256(<8 x float> %x0, i32 66, <8 x float> %x2, i8 -1) + %res2 = fadd <8 x float> %res, %res1 + ret <8 x float> %res2 +} + +declare <8 x float> @llvm.x86.avx512.mask.shuf.f32x4.256(<8 x float>, <8 x float>, i32, <8 x float>, i8) + +define <8 x float>@test_int_x86_avx512_mask_shuf_f32x4_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_shuf_f32x4_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vshuff32x4 $22, %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; CHECK-NEXT: vshuff32x4 $22, %ymm1, %ymm0, %ymm3 {%k1} {z} +; CHECK-NEXT: ## ymm3 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; CHECK-NEXT: vshuff32x4 $22, %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: ## ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; CHECK-NEXT: vaddps %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: vaddps %ymm0, %ymm3, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.avx512.mask.shuf.f32x4.256(<8 x float> %x0, <8 x float> %x1, i32 22, <8 x float> %x3, i8 %x4) + %res1 = call <8 x float> @llvm.x86.avx512.mask.shuf.f32x4.256(<8 x float> %x0, <8 x float> %x1, i32 22, <8 x float> %x3, i8 -1) + %res2 = call <8 x float> @llvm.x86.avx512.mask.shuf.f32x4.256(<8 x float> %x0, <8 x float> %x1, i32 22, <8 x float> zeroinitializer, i8 %x4) + %res3 = fadd <8 x float> %res, %res1 + %res4 = fadd <8 x float> %res2, %res3 + ret <8 x float> %res4 +} + +declare <4 x double> @llvm.x86.avx512.mask.shuf.f64x2.256(<4 x double>, <4 x double>, i32, <4 x double>, i8) + +define <4 x double>@test_int_x86_avx512_mask_shuf_f64x2_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_shuf_f64x2_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vshuff64x2 $22, %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm0[0,1],ymm1[2,3] +; CHECK-NEXT: vshuff64x2 $22, %ymm1, %ymm0, %ymm3 {%k1} {z} +; CHECK-NEXT: ## ymm3 = ymm0[0,1],ymm1[2,3] +; CHECK-NEXT: vshuff64x2 $22, %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: ## ymm0 = ymm0[0,1],ymm1[2,3] +; CHECK-NEXT: vaddpd %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: vaddpd %ymm0, %ymm3, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.avx512.mask.shuf.f64x2.256(<4 x double> %x0, <4 x double> %x1, i32 22, <4 x double> %x3, i8 %x4) + %res1 = call <4 x double> @llvm.x86.avx512.mask.shuf.f64x2.256(<4 x double> %x0, <4 x double> %x1, i32 22, <4 x double> %x3, i8 -1) + %res2 = call <4 x double> @llvm.x86.avx512.mask.shuf.f64x2.256(<4 x double> %x0, <4 x double> %x1, i32 22, <4 x double> zeroinitializer, i8 %x4) + %res3 = fadd <4 x double> %res, %res1 + %res4 = fadd <4 x double> %res2, %res3 + ret <4 x double> %res4 +} + +declare <8 x i32> @llvm.x86.avx512.mask.shuf.i32x4.256(<8 x i32>, <8 x i32>, i32, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_mask_shuf_i32x4_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_shuf_i32x4_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vshufi32x4 $22, %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; CHECK-NEXT: vshufi32x4 $22, %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: ## ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; CHECK-NEXT: vpaddd %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.mask.shuf.i32x4.256(<8 x i32> %x0, <8 x i32> %x1, i32 22, <8 x i32> %x3, i8 %x4) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.shuf.i32x4.256(<8 x i32> %x0, <8 x i32> %x1, i32 22, <8 x i32> %x3, i8 -1) + %res2 = add <8 x i32> %res, %res1 + ret <8 x i32> %res2 +} + +declare <4 x i64> @llvm.x86.avx512.mask.shuf.i64x2.256(<4 x i64>, <4 x i64>, i32, <4 x i64>, i8) + +define <4 x i64>@test_int_x86_avx512_mask_shuf_i64x2_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_shuf_i64x2_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vshufi64x2 $22, %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm0[0,1],ymm1[2,3] +; CHECK-NEXT: vshufi64x2 $22, %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: ## ymm0 = ymm0[0,1],ymm1[2,3] +; CHECK-NEXT: vpaddq %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.mask.shuf.i64x2.256(<4 x i64> %x0, <4 x i64> %x1, i32 22, <4 x i64> %x3, i8 %x4) + %res1 = call <4 x i64> @llvm.x86.avx512.mask.shuf.i64x2.256(<4 x i64> %x0, <4 x i64> %x1, i32 22, <4 x i64> %x3, i8 -1) + %res2 = add <4 x i64> %res, %res1 + ret <4 x i64> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.vextractf32x4.256(<8 x float>, i32, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_vextractf32x4_256(<8 x float> %x0, <4 x float> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vextractf32x4_256: +; CHECK: ## BB#0: +; CHECK-NEXT: kmovw %edi, %k1 +; CHECK-NEXT: vextractf32x4 $1, %ymm0, %xmm1 {%k1} +; CHECK-NEXT: vextractf32x4 $1, %ymm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vextractf32x4 $1, %ymm0, %xmm0 +; CHECK-NEXT: vaddps %xmm2, %xmm1, %xmm1 +; CHECK-NEXT: vaddps %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.vextractf32x4.256(<8 x float> %x0, i32 1, <4 x float> %x2, i8 %x3) + %res1 = call <4 x float> @llvm.x86.avx512.mask.vextractf32x4.256(<8 x float> %x0, i32 1, <4 x float> zeroinitializer, i8 %x3) + %res2 = call <4 x float> @llvm.x86.avx512.mask.vextractf32x4.256(<8 x float> %x0, i32 1, <4 x float> zeroinitializer, i8 -1) + %res3 = fadd <4 x float> %res, %res1 + %res4 = fadd <4 x float> %res2, %res3 + ret <4 x float> %res4 +} + +declare <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double>, i32, <2 x double>, i8) + +define <2 x double>@test_int_x86_avx512_mask_getmant_pd_128(<2 x double> %x0, <2 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_getmant_pd_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vgetmantpd $11, %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vgetmantpd $11, %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vgetmantpd $11, %xmm0, %xmm0 +; CHECK-NEXT: vaddpd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: vaddpd %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double> %x0, i32 11, <2 x double> %x2, i8 %x3) + %res2 = call <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double> %x0, i32 11, <2 x double> zeroinitializer, i8 %x3) + %res1 = call <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double> %x0, i32 11, <2 x double> %x2, i8 -1) + %res3 = fadd <2 x double> %res, %res1 + %res4 = fadd <2 x double> %res2, %res3 + ret <2 x double> %res4 +} + +declare <4 x double> @llvm.x86.avx512.mask.getmant.pd.256(<4 x double>, i32, <4 x double>, i8) + +define <4 x double>@test_int_x86_avx512_mask_getmant_pd_256(<4 x double> %x0, <4 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_getmant_pd_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vgetmantpd $11, %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vgetmantpd $11, %ymm0, %ymm0 +; CHECK-NEXT: vaddpd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.avx512.mask.getmant.pd.256(<4 x double> %x0, i32 11, <4 x double> %x2, i8 %x3) + %res1 = call <4 x double> @llvm.x86.avx512.mask.getmant.pd.256(<4 x double> %x0, i32 11, <4 x double> %x2, i8 -1) + %res2 = fadd <4 x double> %res, %res1 + ret <4 x double> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.getmant.ps.128(<4 x float>, i32, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_getmant_ps_128(<4 x float> %x0, <4 x float> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_getmant_ps_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vgetmantps $11, %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vgetmantps $11, %xmm0, %xmm0 +; CHECK-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.getmant.ps.128(<4 x float> %x0, i32 11, <4 x float> %x2, i8 %x3) + %res1 = call <4 x float> @llvm.x86.avx512.mask.getmant.ps.128(<4 x float> %x0, i32 11, <4 x float> %x2, i8 -1) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <8 x float> @llvm.x86.avx512.mask.getmant.ps.256(<8 x float>, i32, <8 x float>, i8) + +define <8 x float>@test_int_x86_avx512_mask_getmant_ps_256(<8 x float> %x0, <8 x float> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_getmant_ps_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vgetmantps $11, %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vgetmantps $11, %ymm0, %ymm0 +; CHECK-NEXT: vaddps %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.avx512.mask.getmant.ps.256(<8 x float> %x0, i32 11, <8 x float> %x2, i8 %x3) + %res1 = call <8 x float> @llvm.x86.avx512.mask.getmant.ps.256(<8 x float> %x0, i32 11, <8 x float> %x2, i8 -1) + %res2 = fadd <8 x float> %res, %res1 + ret <8 x float> %res2 +} + +declare <2 x double> @llvm.x86.avx512.mask.shuf.pd.128(<2 x double>, <2 x double>, i32, <2 x double>, i8) + +define <2 x double>@test_int_x86_avx512_mask_shuf_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_shuf_pd_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vshufpd $22, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: ## xmm2 = xmm2[0],k1[1] +; CHECK-NEXT: vshufpd $22, %xmm1, %xmm0, %xmm3 {%k1} {z} +; CHECK-NEXT: ## xmm3 = k1[0],xmm0[1] +; CHECK-NEXT: vshufpd $22, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: ## xmm0 = xmm0[0],xmm1[1] +; CHECK-NEXT: vaddpd %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: vaddpd %xmm0, %xmm3, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.shuf.pd.128(<2 x double> %x0, <2 x double> %x1, i32 22, <2 x double> %x3, i8 %x4) + %res1 = call <2 x double> @llvm.x86.avx512.mask.shuf.pd.128(<2 x double> %x0, <2 x double> %x1, i32 22, <2 x double> %x3, i8 -1) + %res2 = call <2 x double> @llvm.x86.avx512.mask.shuf.pd.128(<2 x double> %x0, <2 x double> %x1, i32 22, <2 x double> zeroinitializer, i8 %x4) + %res3 = fadd <2 x double> %res, %res1 + %res4 = fadd <2 x double> %res2, %res3 + ret <2 x double> %res4 +} + +declare <4 x double> @llvm.x86.avx512.mask.shuf.pd.256(<4 x double>, <4 x double>, i32, <4 x double>, i8) + +define <4 x double>@test_int_x86_avx512_mask_shuf_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_shuf_pd_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vshufpd $22, %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm2[0],k1[1],ymm2[3],k1[2] +; CHECK-NEXT: vshufpd $22, %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: ## ymm0 = ymm0[0],ymm1[1],ymm0[3],ymm1[2] +; CHECK-NEXT: vaddpd %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.avx512.mask.shuf.pd.256(<4 x double> %x0, <4 x double> %x1, i32 22, <4 x double> %x3, i8 %x4) + %res1 = call <4 x double> @llvm.x86.avx512.mask.shuf.pd.256(<4 x double> %x0, <4 x double> %x1, i32 22, <4 x double> %x3, i8 -1) + %res2 = fadd <4 x double> %res, %res1 + ret <4 x double> %res2 +} + +declare <4 x float> @llvm.x86.avx512.mask.shuf.ps.128(<4 x float>, <4 x float>, i32, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_shuf_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_shuf_ps_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vshufps $22, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: ## xmm2 = xmm2[2,1],k1[1,0] +; CHECK-NEXT: vshufps $22, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: ## xmm0 = xmm0[2,1],xmm1[1,0] +; CHECK-NEXT: vaddps %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.shuf.ps.128(<4 x float> %x0, <4 x float> %x1, i32 22, <4 x float> %x3, i8 %x4) + %res1 = call <4 x float> @llvm.x86.avx512.mask.shuf.ps.128(<4 x float> %x0, <4 x float> %x1, i32 22, <4 x float> %x3, i8 -1) + %res2 = fadd <4 x float> %res, %res1 + ret <4 x float> %res2 +} + +declare <8 x float> @llvm.x86.avx512.mask.shuf.ps.256(<8 x float>, <8 x float>, i32, <8 x float>, i8) + +define <8 x float>@test_int_x86_avx512_mask_shuf_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_shuf_ps_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vshufps $22, %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: ## ymm2 = ymm2[2,1],k1[1,0],ymm2[6,5],k1[5,4] +; CHECK-NEXT: vshufps $22, %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: ## ymm0 = ymm0[2,1],ymm1[1,0],ymm0[6,5],ymm1[5,4] +; CHECK-NEXT: vaddps %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.avx512.mask.shuf.ps.256(<8 x float> %x0, <8 x float> %x1, i32 22, <8 x float> %x3, i8 %x4) + %res1 = call <8 x float> @llvm.x86.avx512.mask.shuf.ps.256(<8 x float> %x0, <8 x float> %x1, i32 22, <8 x float> %x3, i8 -1) + %res2 = fadd <8 x float> %res, %res1 + ret <8 x float> %res2 +} + +declare <4 x i32> @llvm.x86.avx512.mask.valign.d.128(<4 x i32>, <4 x i32>, i32, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_mask_valign_d_128(<4 x i32> %x0, <4 x i32> %x1,<4 x i32> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_valign_d_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: valignd $22, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: valignd $22, %xmm1, %xmm0, %xmm3 {%k1} {z} +; CHECK-NEXT: valignd $22, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: vpaddd %xmm3, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.valign.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22, <4 x i32> %x3, i8 %x4) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.valign.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22, <4 x i32> %x3, i8 -1) + %res2 = call <4 x i32> @llvm.x86.avx512.mask.valign.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22, <4 x i32> zeroinitializer,i8 %x4) + %res3 = add <4 x i32> %res, %res1 + %res4 = add <4 x i32> %res3, %res2 + ret <4 x i32> %res4 +} + +declare <8 x i32> @llvm.x86.avx512.mask.valign.d.256(<8 x i32>, <8 x i32>, i32, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_mask_valign_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_valign_d_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: valignd $22, %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: valignd $22, %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.mask.valign.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22, <8 x i32> %x3, i8 %x4) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.valign.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22, <8 x i32> %x3, i8 -1) + %res2 = add <8 x i32> %res, %res1 + ret <8 x i32> %res2 +} + +declare <2 x i64> @llvm.x86.avx512.mask.valign.q.128(<2 x i64>, <2 x i64>, i32, <2 x i64>, i8) + +define <2 x i64>@test_int_x86_avx512_mask_valign_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_valign_q_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: valignq $22, %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: valignq $22, %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddq %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x i64> @llvm.x86.avx512.mask.valign.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22, <2 x i64> %x3, i8 %x4) + %res1 = call <2 x i64> @llvm.x86.avx512.mask.valign.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22, <2 x i64> %x3, i8 -1) + %res2 = add <2 x i64> %res, %res1 + ret <2 x i64> %res2 +} + +declare <4 x i64> @llvm.x86.avx512.mask.valign.q.256(<4 x i64>, <4 x i64>, i32, <4 x i64>, i8) + +define <4 x i64>@test_int_x86_avx512_mask_valign_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_valign_q_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: valignq $22, %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: valignq $22, %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddq %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.mask.valign.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22, <4 x i64> %x3, i8 %x4) + %res1 = call <4 x i64> @llvm.x86.avx512.mask.valign.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22, <4 x i64> %x3, i8 -1) + %res2 = add <4 x i64> %res, %res1 + ret <4 x i64> %res2 +} + +declare <4 x double> @llvm.x86.avx512.mask.vpermil.pd.256(<4 x double>, i32, <4 x double>, i8) + +define <4 x double>@test_int_x86_avx512_mask_vpermil_pd_256(<4 x double> %x0, <4 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpermil_pd_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpermilpd $22, %ymm0, %ymm1 {%k1} +; CHECK-NEXT: ## ymm1 = ymm1[0,1,3,2] +; CHECK-NEXT: vpermilpd $22, %ymm0, %ymm2 {%k1} {z} +; CHECK-NEXT: ## ymm2 = k1[0,1,3,2] +; CHECK-NEXT: vpermilpd $22, %ymm0, %ymm0 +; CHECK-NEXT: ## ymm0 = ymm0[0,1,3,2] +; CHECK-NEXT: vaddpd %ymm2, %ymm1, %ymm1 +; CHECK-NEXT: vaddpd %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.avx512.mask.vpermil.pd.256(<4 x double> %x0, i32 22, <4 x double> %x2, i8 %x3) + %res1 = call <4 x double> @llvm.x86.avx512.mask.vpermil.pd.256(<4 x double> %x0, i32 22, <4 x double> zeroinitializer, i8 %x3) + %res2 = call <4 x double> @llvm.x86.avx512.mask.vpermil.pd.256(<4 x double> %x0, i32 22, <4 x double> %x2, i8 -1) + %res3 = fadd <4 x double> %res, %res1 + %res4 = fadd <4 x double> %res2, %res3 + ret <4 x double> %res4 +} + +declare <2 x double> @llvm.x86.avx512.mask.vpermil.pd.128(<2 x double>, i32, <2 x double>, i8) + +define <2 x double>@test_int_x86_avx512_mask_vpermil_pd_128(<2 x double> %x0, <2 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpermil_pd_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpermilpd $1, %xmm0, %xmm1 {%k1} +; CHECK-NEXT: ## xmm1 = xmm1[1,0] +; CHECK-NEXT: vpermilpd $1, %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: ## xmm2 = k1[1,0] +; CHECK-NEXT: vpermilpd $1, %xmm0, %xmm0 +; CHECK-NEXT: ## xmm0 = xmm0[1,0] +; CHECK-NEXT: vaddpd %xmm2, %xmm1, %xmm1 +; CHECK-NEXT: vaddpd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.vpermil.pd.128(<2 x double> %x0, i32 1, <2 x double> %x2, i8 %x3) + %res1 = call <2 x double> @llvm.x86.avx512.mask.vpermil.pd.128(<2 x double> %x0, i32 1, <2 x double> zeroinitializer, i8 %x3) + %res2 = call <2 x double> @llvm.x86.avx512.mask.vpermil.pd.128(<2 x double> %x0, i32 1, <2 x double> %x2, i8 -1) + %res3 = fadd <2 x double> %res, %res1 + %res4 = fadd <2 x double> %res3, %res2 + ret <2 x double> %res4 +} + +declare <8 x float> @llvm.x86.avx512.mask.vpermil.ps.256(<8 x float>, i32, <8 x float>, i8) + +define <8 x float>@test_int_x86_avx512_mask_vpermil_ps_256(<8 x float> %x0, <8 x float> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpermil_ps_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpermilps $22, %ymm0, %ymm1 {%k1} +; CHECK-NEXT: ## ymm1 = ymm1[2,1,1,0,6,5,5,4] +; CHECK-NEXT: vpermilps $22, %ymm0, %ymm2 {%k1} {z} +; CHECK-NEXT: ## ymm2 = k1[2,1,1,0,6,5,5,4] +; CHECK-NEXT: vpermilps $22, %ymm0, %ymm0 +; CHECK-NEXT: ## ymm0 = ymm0[2,1,1,0,6,5,5,4] +; CHECK-NEXT: vaddps %ymm2, %ymm1, %ymm1 +; CHECK-NEXT: vaddps %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.avx512.mask.vpermil.ps.256(<8 x float> %x0, i32 22, <8 x float> %x2, i8 %x3) + %res1 = call <8 x float> @llvm.x86.avx512.mask.vpermil.ps.256(<8 x float> %x0, i32 22, <8 x float> zeroinitializer, i8 %x3) + %res2 = call <8 x float> @llvm.x86.avx512.mask.vpermil.ps.256(<8 x float> %x0, i32 22, <8 x float> %x2, i8 -1) + %res3 = fadd <8 x float> %res, %res1 + %res4 = fadd <8 x float> %res3, %res2 + ret <8 x float> %res4 +} + +declare <4 x float> @llvm.x86.avx512.mask.vpermil.ps.128(<4 x float>, i32, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_vpermil_ps_128(<4 x float> %x0, <4 x float> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpermil_ps_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpermilps $22, %xmm0, %xmm1 {%k1} +; CHECK-NEXT: ## xmm1 = xmm1[2,1,1,0] +; CHECK-NEXT: vpermilps $22, %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: ## xmm2 = k1[2,1,1,0] +; CHECK-NEXT: vpermilps $22, %xmm0, %xmm0 +; CHECK-NEXT: ## xmm0 = xmm0[2,1,1,0] +; CHECK-NEXT: vaddps %xmm2, %xmm1, %xmm1 +; CHECK-NEXT: vaddps %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.vpermil.ps.128(<4 x float> %x0, i32 22, <4 x float> %x2, i8 %x3) + %res1 = call <4 x float> @llvm.x86.avx512.mask.vpermil.ps.128(<4 x float> %x0, i32 22, <4 x float> zeroinitializer, i8 %x3) + %res2 = call <4 x float> @llvm.x86.avx512.mask.vpermil.ps.128(<4 x float> %x0, i32 22, <4 x float> %x2, i8 -1) + %res3 = fadd <4 x float> %res, %res1 + %res4 = fadd <4 x float> %res2, %res3 + ret <4 x float> %res4 +} + +declare <4 x double> @llvm.x86.avx512.mask.vpermilvar.pd.256(<4 x double>, <4 x i64>, <4 x double>, i8) + +define <4 x double>@test_int_x86_avx512_mask_vpermilvar_pd_256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpermilvar_pd_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpermilpd %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: vpermilpd %ymm1, %ymm0, %ymm3 {%k1} {z} +; CHECK-NEXT: vpermilpd %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vaddpd %ymm3, %ymm2, %ymm1 +; CHECK-NEXT: vaddpd %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.avx512.mask.vpermilvar.pd.256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2, i8 %x3) + %res1 = call <4 x double> @llvm.x86.avx512.mask.vpermilvar.pd.256(<4 x double> %x0, <4 x i64> %x1, <4 x double> zeroinitializer, i8 %x3) + %res2 = call <4 x double> @llvm.x86.avx512.mask.vpermilvar.pd.256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2, i8 -1) + %res3 = fadd <4 x double> %res, %res1 + %res4 = fadd <4 x double> %res2, %res3 + ret <4 x double> %res4 +} + +declare <2 x double> @llvm.x86.avx512.mask.vpermilvar.pd.128(<2 x double>, <2 x i64>, <2 x double>, i8) + +define <2 x double>@test_int_x86_avx512_mask_vpermilvar_pd_128(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpermilvar_pd_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpermilpd %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vpermilpd %xmm1, %xmm0, %xmm3 {%k1} {z} +; CHECK-NEXT: vpermilpd %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vaddpd %xmm3, %xmm2, %xmm1 +; CHECK-NEXT: vaddpd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.vpermilvar.pd.128(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2, i8 %x3) + %res1 = call <2 x double> @llvm.x86.avx512.mask.vpermilvar.pd.128(<2 x double> %x0, <2 x i64> %x1, <2 x double> zeroinitializer, i8 %x3) + %res2 = call <2 x double> @llvm.x86.avx512.mask.vpermilvar.pd.128(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2, i8 -1) + %res3 = fadd <2 x double> %res, %res1 + %res4 = fadd <2 x double> %res3, %res2 + ret <2 x double> %res4 +} + +declare <8 x float> @llvm.x86.avx512.mask.vpermilvar.ps.256(<8 x float>, <8 x i32>, <8 x float>, i8) + +define <8 x float>@test_int_x86_avx512_mask_vpermilvar_ps_256(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpermilvar_ps_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpermilps %ymm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: vpermilps %ymm1, %ymm0, %ymm3 {%k1} {z} +; CHECK-NEXT: vpermilps %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vaddps %ymm3, %ymm2, %ymm1 +; CHECK-NEXT: vaddps %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.avx512.mask.vpermilvar.ps.256(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2, i8 %x3) + %res1 = call <8 x float> @llvm.x86.avx512.mask.vpermilvar.ps.256(<8 x float> %x0, <8 x i32> %x1, <8 x float> zeroinitializer, i8 %x3) + %res2 = call <8 x float> @llvm.x86.avx512.mask.vpermilvar.ps.256(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2, i8 -1) + %res3 = fadd <8 x float> %res, %res1 + %res4 = fadd <8 x float> %res3, %res2 + ret <8 x float> %res4 +} + +declare <4 x float> @llvm.x86.avx512.mask.vpermilvar.ps.128(<4 x float>, <4 x i32>, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_vpermilvar_ps_128(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2, i8 %x3) { +; CHECK-LABEL: test_int_x86_avx512_mask_vpermilvar_ps_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpermilps %xmm1, %xmm0, %xmm2 {%k1} +; CHECK-NEXT: vpermilps %xmm1, %xmm0, %xmm3 {%k1} {z} +; CHECK-NEXT: vpermilps %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vaddps %xmm3, %xmm2, %xmm1 +; CHECK-NEXT: vaddps %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.vpermilvar.ps.128(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2, i8 %x3) + %res1 = call <4 x float> @llvm.x86.avx512.mask.vpermilvar.ps.128(<4 x float> %x0, <4 x i32> %x1, <4 x float> zeroinitializer, i8 %x3) + %res2 = call <4 x float> @llvm.x86.avx512.mask.vpermilvar.ps.128(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2, i8 -1) + %res3 = fadd <4 x float> %res, %res1 + %res4 = fadd <4 x float> %res2, %res3 + ret <4 x float> %res4 +} + +declare <8 x float> @llvm.x86.avx512.mask.insertf32x4.256(<8 x float>, <4 x float>, i32, <8 x float>, i8) + +define <8 x float>@test_int_x86_avx512_mask_insertf32x4_256(<8 x float> %x0, <4 x float> %x1, <8 x float> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_insertf32x4_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vinsertf32x4 $1, %xmm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: vinsertf32x4 $1, %xmm1, %ymm0, %ymm3 {%k1} {z} +; CHECK-NEXT: vinsertf32x4 $1, %xmm1, %ymm0, %ymm0 +; CHECK-NEXT: vaddps %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: vaddps %ymm0, %ymm3, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.avx512.mask.insertf32x4.256(<8 x float> %x0, <4 x float> %x1, i32 1, <8 x float> %x3, i8 %x4) + %res1 = call <8 x float> @llvm.x86.avx512.mask.insertf32x4.256(<8 x float> %x0, <4 x float> %x1, i32 1, <8 x float> %x3, i8 -1) + %res2 = call <8 x float> @llvm.x86.avx512.mask.insertf32x4.256(<8 x float> %x0, <4 x float> %x1, i32 1, <8 x float> zeroinitializer, i8 %x4) + %res3 = fadd <8 x float> %res, %res1 + %res4 = fadd <8 x float> %res2, %res3 + ret <8 x float> %res4 +} + +declare <8 x i32> @llvm.x86.avx512.mask.inserti32x4.256(<8 x i32>, <4 x i32>, i32, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_mask_inserti32x4_256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> %x3, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_inserti32x4_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vinserti32x4 $1, %xmm1, %ymm0, %ymm2 {%k1} +; CHECK-NEXT: vinserti32x4 $1, %xmm1, %ymm0, %ymm3 {%k1} {z} +; CHECK-NEXT: vinserti32x4 $1, %xmm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: vpaddd %ymm0, %ymm3, %ymm0 +; CHECK-NEXT: retq + + %res = call <8 x i32> @llvm.x86.avx512.mask.inserti32x4.256(<8 x i32> %x0, <4 x i32> %x1, i32 1, <8 x i32> %x3, i8 %x4) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.inserti32x4.256(<8 x i32> %x0, <4 x i32> %x1, i32 1, <8 x i32> %x3, i8 -1) + %res2 = call <8 x i32> @llvm.x86.avx512.mask.inserti32x4.256(<8 x i32> %x0, <4 x i32> %x1, i32 1, <8 x i32> zeroinitializer, i8 %x4) + %res3 = add <8 x i32> %res, %res1 + %res4 = add <8 x i32> %res2, %res3 + ret <8 x i32> %res4 +} + +declare <4 x i32> @llvm.x86.avx512.mask.pternlog.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i32, i8) + +define <4 x i32>@test_int_x86_avx512_mask_pternlog_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_pternlog_d_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovaps %zmm0, %zmm3 +; CHECK-NEXT: vpternlogd $33, %xmm2, %xmm1, %xmm3 {%k1} +; CHECK-NEXT: vpternlogd $33, %xmm2, %xmm1, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm3, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.mask.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33, i8 %x4) + %res1 = call <4 x i32> @llvm.x86.avx512.mask.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33, i8 -1) + %res2 = add <4 x i32> %res, %res1 + ret <4 x i32> %res2 +} + +declare <4 x i32> @llvm.x86.avx512.maskz.pternlog.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i32, i8) + +define <4 x i32>@test_int_x86_avx512_maskz_pternlog_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_maskz_pternlog_d_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovaps %zmm0, %zmm3 +; CHECK-NEXT: vpternlogd $33, %xmm2, %xmm1, %xmm3 {%k1} {z} +; CHECK-NEXT: vpternlogd $33, %xmm2, %xmm1, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm3, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.maskz.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33, i8 %x4) + %res1 = call <4 x i32> @llvm.x86.avx512.maskz.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33, i8 -1) + %res2 = add <4 x i32> %res, %res1 + ret <4 x i32> %res2 +} + +declare <8 x i32> @llvm.x86.avx512.mask.pternlog.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i32, i8) + +define <8 x i32>@test_int_x86_avx512_mask_pternlog_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_pternlog_d_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovaps %zmm0, %zmm3 +; CHECK-NEXT: vpternlogd $33, %ymm2, %ymm1, %ymm3 {%k1} +; CHECK-NEXT: vpternlogd $33, %ymm2, %ymm1, %ymm0 +; CHECK-NEXT: vpaddd %ymm0, %ymm3, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.mask.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33, i8 %x4) + %res1 = call <8 x i32> @llvm.x86.avx512.mask.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33, i8 -1) + %res2 = add <8 x i32> %res, %res1 + ret <8 x i32> %res2 +} + +declare <8 x i32> @llvm.x86.avx512.maskz.pternlog.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i32, i8) + +define <8 x i32>@test_int_x86_avx512_maskz_pternlog_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_maskz_pternlog_d_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovaps %zmm0, %zmm3 +; CHECK-NEXT: vpternlogd $33, %ymm2, %ymm1, %ymm3 {%k1} {z} +; CHECK-NEXT: vpternlogd $33, %ymm2, %ymm1, %ymm0 +; CHECK-NEXT: vpaddd %ymm0, %ymm3, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.maskz.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33, i8 %x4) + %res1 = call <8 x i32> @llvm.x86.avx512.maskz.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33, i8 -1) + %res2 = add <8 x i32> %res, %res1 + ret <8 x i32> %res2 +} + +declare <2 x i64> @llvm.x86.avx512.mask.pternlog.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i32, i8) + +define <2 x i64>@test_int_x86_avx512_mask_pternlog_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_pternlog_q_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovaps %zmm0, %zmm3 +; CHECK-NEXT: vpternlogq $33, %xmm2, %xmm1, %xmm3 {%k1} +; CHECK-NEXT: vpternlogq $33, %xmm2, %xmm1, %xmm0 +; CHECK-NEXT: vpaddq %xmm0, %xmm3, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x i64> @llvm.x86.avx512.mask.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33, i8 %x4) + %res1 = call <2 x i64> @llvm.x86.avx512.mask.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33, i8 -1) + %res2 = add <2 x i64> %res, %res1 + ret <2 x i64> %res2 +} + +declare <2 x i64> @llvm.x86.avx512.maskz.pternlog.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i32, i8) + +define <2 x i64>@test_int_x86_avx512_maskz_pternlog_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_maskz_pternlog_q_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovaps %zmm0, %zmm3 +; CHECK-NEXT: vpternlogq $33, %xmm2, %xmm1, %xmm3 {%k1} {z} +; CHECK-NEXT: vpternlogq $33, %xmm2, %xmm1, %xmm0 +; CHECK-NEXT: vpaddq %xmm0, %xmm3, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x i64> @llvm.x86.avx512.maskz.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33, i8 %x4) + %res1 = call <2 x i64> @llvm.x86.avx512.maskz.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33, i8 -1) + %res2 = add <2 x i64> %res, %res1 + ret <2 x i64> %res2 +} + +declare <4 x i64> @llvm.x86.avx512.mask.pternlog.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i32, i8) + +define <4 x i64>@test_int_x86_avx512_mask_pternlog_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_mask_pternlog_q_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovaps %zmm0, %zmm3 +; CHECK-NEXT: vpternlogq $33, %ymm2, %ymm1, %ymm3 {%k1} +; CHECK-NEXT: vpternlogq $33, %ymm2, %ymm1, %ymm0 +; CHECK-NEXT: vpaddq %ymm0, %ymm3, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.mask.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33, i8 %x4) + %res1 = call <4 x i64> @llvm.x86.avx512.mask.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33, i8 -1) + %res2 = add <4 x i64> %res, %res1 + ret <4 x i64> %res2 +} + +declare <4 x i64> @llvm.x86.avx512.maskz.pternlog.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i32, i8) + +define <4 x i64>@test_int_x86_avx512_maskz_pternlog_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x4) { +; CHECK-LABEL: test_int_x86_avx512_maskz_pternlog_q_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovaps %zmm0, %zmm3 +; CHECK-NEXT: vpternlogq $33, %ymm2, %ymm1, %ymm3 {%k1} {z} +; CHECK-NEXT: vpternlogq $33, %ymm2, %ymm1, %ymm0 +; CHECK-NEXT: vpaddq %ymm0, %ymm3, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.maskz.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33, i8 %x4) + %res1 = call <4 x i64> @llvm.x86.avx512.maskz.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33, i8 -1) + %res2 = add <4 x i64> %res, %res1 + ret <4 x i64> %res2 +} + +declare <8 x i32> @llvm.x86.avx512.pbroadcastd.256(<4 x i32>, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_pbroadcastd_256(<4 x i32> %x0, <8 x i32> %x1, i8 %mask) { +; CHECK-LABEL: test_int_x86_avx512_pbroadcastd_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpbroadcastd %xmm0, %ymm1 {%k1} +; CHECK-NEXT: vpbroadcastd %xmm0, %ymm2 {%k1} {z} +; CHECK-NEXT: vpbroadcastd %xmm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddd %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x i32> @llvm.x86.avx512.pbroadcastd.256(<4 x i32> %x0, <8 x i32> %x1, i8 -1) + %res1 = call <8 x i32> @llvm.x86.avx512.pbroadcastd.256(<4 x i32> %x0, <8 x i32> %x1, i8 %mask) + %res2 = call <8 x i32> @llvm.x86.avx512.pbroadcastd.256(<4 x i32> %x0, <8 x i32> zeroinitializer, i8 %mask) + %res3 = add <8 x i32> %res, %res1 + %res4 = add <8 x i32> %res2, %res3 + ret <8 x i32> %res4 +} + +declare <4 x i32> @llvm.x86.avx512.pbroadcastd.128(<4 x i32>, <4 x i32>, i8) + +define <4 x i32>@test_int_x86_avx512_pbroadcastd_128(<4 x i32> %x0, <4 x i32> %x1, i8 %mask) { +; CHECK-LABEL: test_int_x86_avx512_pbroadcastd_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpbroadcastd %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpbroadcastd %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpbroadcastd %xmm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddd %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x i32> @llvm.x86.avx512.pbroadcastd.128(<4 x i32> %x0, <4 x i32> %x1, i8 -1) + %res1 = call <4 x i32> @llvm.x86.avx512.pbroadcastd.128(<4 x i32> %x0, <4 x i32> %x1, i8 %mask) + %res2 = call <4 x i32> @llvm.x86.avx512.pbroadcastd.128(<4 x i32> %x0, <4 x i32> zeroinitializer, i8 %mask) + %res3 = add <4 x i32> %res, %res1 + %res4 = add <4 x i32> %res2, %res3 + ret <4 x i32> %res4 +} + +declare <4 x i64> @llvm.x86.avx512.pbroadcastq.256(<2 x i64>, <4 x i64>, i8) + +define <4 x i64>@test_int_x86_avx512_pbroadcastq_256(<2 x i64> %x0, <4 x i64> %x1, i8 %mask) { +; CHECK-LABEL: test_int_x86_avx512_pbroadcastq_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpbroadcastq %xmm0, %ymm1 {%k1} +; CHECK-NEXT: vpbroadcastq %xmm0, %ymm2 {%k1} {z} +; CHECK-NEXT: vpbroadcastq %xmm0, %ymm0 +; CHECK-NEXT: vpaddq %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: vpaddq %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x i64> @llvm.x86.avx512.pbroadcastq.256(<2 x i64> %x0, <4 x i64> %x1,i8 -1) + %res1 = call <4 x i64> @llvm.x86.avx512.pbroadcastq.256(<2 x i64> %x0, <4 x i64> %x1,i8 %mask) + %res2 = call <4 x i64> @llvm.x86.avx512.pbroadcastq.256(<2 x i64> %x0, <4 x i64> zeroinitializer,i8 %mask) + %res3 = add <4 x i64> %res, %res1 + %res4 = add <4 x i64> %res2, %res3 + ret <4 x i64> %res4 +} + +declare <2 x i64> @llvm.x86.avx512.pbroadcastq.128(<2 x i64>, <2 x i64>, i8) + +define <2 x i64>@test_int_x86_avx512_pbroadcastq_128(<2 x i64> %x0, <2 x i64> %x1, i8 %mask) { +; CHECK-LABEL: test_int_x86_avx512_pbroadcastq_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vpbroadcastq %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vpbroadcastq %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vpbroadcastq %xmm0, %xmm0 +; CHECK-NEXT: vpaddq %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vpaddq %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x i64> @llvm.x86.avx512.pbroadcastq.128(<2 x i64> %x0, <2 x i64> %x1,i8 -1) + %res1 = call <2 x i64> @llvm.x86.avx512.pbroadcastq.128(<2 x i64> %x0, <2 x i64> %x1,i8 %mask) + %res2 = call <2 x i64> @llvm.x86.avx512.pbroadcastq.128(<2 x i64> %x0, <2 x i64> zeroinitializer,i8 %mask) + %res3 = add <2 x i64> %res, %res1 + %res4 = add <2 x i64> %res2, %res3 + ret <2 x i64> %res4 +} + +define <4 x float> @test_x86_vcvtph2ps_128(<8 x i16> %a0) { + ; CHECK: test_x86_vcvtph2ps_128 + ; CHECK: vcvtph2ps %xmm0, %xmm0 + %res = call <4 x float> @llvm.x86.avx512.mask.vcvtph2ps.128(<8 x i16> %a0, <4 x float> zeroinitializer, i8 -1) + ret <4 x float> %res +} + +define <4 x float> @test_x86_vcvtph2ps_128_rrk(<8 x i16> %a0,<4 x float> %a1, i8 %mask) { + ; CHECK: test_x86_vcvtph2ps_128_rrk + ; CHECK: vcvtph2ps %xmm0, %xmm1 {%k1} + %res = call <4 x float> @llvm.x86.avx512.mask.vcvtph2ps.128(<8 x i16> %a0, <4 x float> %a1, i8 %mask) + ret <4 x float> %res +} + + +define <4 x float> @test_x86_vcvtph2ps_128_rrkz(<8 x i16> %a0, i8 %mask) { + ; CHECK: test_x86_vcvtph2ps_128_rrkz + ; CHECK: vcvtph2ps %xmm0, %xmm0 {%k1} {z} + %res = call <4 x float> @llvm.x86.avx512.mask.vcvtph2ps.128(<8 x i16> %a0, <4 x float> zeroinitializer, i8 %mask) + ret <4 x float> %res +} + +declare <4 x float> @llvm.x86.avx512.mask.vcvtph2ps.128(<8 x i16>, <4 x float>, i8) nounwind readonly + +define <8 x float> @test_x86_vcvtph2ps_256(<8 x i16> %a0) { + ; CHECK: test_x86_vcvtph2ps_256 + ; CHECK: vcvtph2ps %xmm0, %ymm0 + %res = call <8 x float> @llvm.x86.avx512.mask.vcvtph2ps.256(<8 x i16> %a0, <8 x float> zeroinitializer, i8 -1) + ret <8 x float> %res +} + +define <8 x float> @test_x86_vcvtph2ps_256_rrk(<8 x i16> %a0,<8 x float> %a1, i8 %mask) { + ; CHECK: test_x86_vcvtph2ps_256_rrk + ; CHECK: vcvtph2ps %xmm0, %ymm1 {%k1} + %res = call <8 x float> @llvm.x86.avx512.mask.vcvtph2ps.256(<8 x i16> %a0, <8 x float> %a1, i8 %mask) + ret <8 x float> %res +} + +define <8 x float> @test_x86_vcvtph2ps_256_rrkz(<8 x i16> %a0, i8 %mask) { + ; CHECK: test_x86_vcvtph2ps_256_rrkz + ; CHECK: vcvtph2ps %xmm0, %ymm0 {%k1} {z} + %res = call <8 x float> @llvm.x86.avx512.mask.vcvtph2ps.256(<8 x i16> %a0, <8 x float> zeroinitializer, i8 %mask) + ret <8 x float> %res +} + +declare <8 x float> @llvm.x86.avx512.mask.vcvtph2ps.256(<8 x i16>, <8 x float>, i8) nounwind readonly + +define <8 x i16> @test_x86_vcvtps2ph_128(<4 x float> %a0) { + ; CHECK: test_x86_vcvtps2ph_128 + ; CHECK: vcvtps2ph $2, %xmm0, %xmm0 + %res = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128(<4 x float> %a0, i32 2, <8 x i16> zeroinitializer, i8 -1) + ret <8 x i16> %res +} + + +declare <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128(<4 x float>, i32, <8 x i16>, i8) nounwind readonly + +define <8 x i16> @test_x86_vcvtps2ph_256(<8 x float> %a0) { + ; CHECK: test_x86_vcvtps2ph_256 + ; CHECK: vcvtps2ph $2, %ymm0, %xmm0 + %res = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256(<8 x float> %a0, i32 2, <8 x i16> zeroinitializer, i8 -1) + ret <8 x i16> %res +} + +declare <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256(<8 x float>, i32, <8 x i16>, i8) nounwind readonly + +declare <4 x float> @llvm.x86.avx512.mask.movsldup.128(<4 x float>, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_movsldup_128(<4 x float> %x0, <4 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_movsldup_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovsldup %xmm0, %xmm1 {%k1} +; CHECK-NEXT: ## xmm1 = xmm0[0,0,2,2] +; CHECK-NEXT: vmovsldup %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: ## xmm2 = xmm0[0,0,2,2] +; CHECK-NEXT: vmovsldup %xmm0, %xmm0 +; CHECK-NEXT: ## xmm0 = xmm0[0,0,2,2] +; CHECK-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: vaddps %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.movsldup.128(<4 x float> %x0, <4 x float> %x1, i8 %x2) + %res1 = call <4 x float> @llvm.x86.avx512.mask.movsldup.128(<4 x float> %x0, <4 x float> %x1, i8 -1) + %res2 = call <4 x float> @llvm.x86.avx512.mask.movsldup.128(<4 x float> %x0, <4 x float> zeroinitializer, i8 %x2) + %res3 = fadd <4 x float> %res, %res1 + %res4 = fadd <4 x float> %res2, %res3 + ret <4 x float> %res4 +} + +declare <8 x float> @llvm.x86.avx512.mask.movsldup.256(<8 x float>, <8 x float>, i8) + +define <8 x float>@test_int_x86_avx512_mask_movsldup_256(<8 x float> %x0, <8 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_movsldup_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovsldup %ymm0, %ymm1 {%k1} +; CHECK-NEXT: ## ymm1 = ymm0[0,0,2,2,4,4,6,6] +; CHECK-NEXT: vmovsldup %ymm0, %ymm2 {%k1} {z} +; CHECK-NEXT: ## ymm2 = ymm0[0,0,2,2,4,4,6,6] +; CHECK-NEXT: vmovsldup %ymm0, %ymm0 +; CHECK-NEXT: ## ymm0 = ymm0[0,0,2,2,4,4,6,6] +; CHECK-NEXT: vaddps %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: vaddps %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.avx512.mask.movsldup.256(<8 x float> %x0, <8 x float> %x1, i8 %x2) + %res1 = call <8 x float> @llvm.x86.avx512.mask.movsldup.256(<8 x float> %x0, <8 x float> %x1, i8 -1) + %res2 = call <8 x float> @llvm.x86.avx512.mask.movsldup.256(<8 x float> %x0, <8 x float> zeroinitializer, i8 %x2) + %res3 = fadd <8 x float> %res, %res1 + %res4 = fadd <8 x float> %res2, %res3 + ret <8 x float> %res4 +} + +declare <4 x float> @llvm.x86.avx512.mask.movshdup.128(<4 x float>, <4 x float>, i8) + +define <4 x float>@test_int_x86_avx512_mask_movshdup_128(<4 x float> %x0, <4 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_movshdup_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovshdup %xmm0, %xmm1 {%k1} +; CHECK-NEXT: ## xmm1 = xmm0[1,1,3,3] +; CHECK-NEXT: vmovshdup %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: ## xmm2 = xmm0[1,1,3,3] +; CHECK-NEXT: vmovshdup %xmm0, %xmm0 +; CHECK-NEXT: ## xmm0 = xmm0[1,1,3,3] +; CHECK-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: vaddps %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.avx512.mask.movshdup.128(<4 x float> %x0, <4 x float> %x1, i8 %x2) + %res1 = call <4 x float> @llvm.x86.avx512.mask.movshdup.128(<4 x float> %x0, <4 x float> %x1, i8 -1) + %res2 = call <4 x float> @llvm.x86.avx512.mask.movshdup.128(<4 x float> %x0, <4 x float> zeroinitializer, i8 %x2) + %res3 = fadd <4 x float> %res, %res1 + %res4 = fadd <4 x float> %res2, %res3 + ret <4 x float> %res4 +} + +declare <8 x float> @llvm.x86.avx512.mask.movshdup.256(<8 x float>, <8 x float>, i8) + +define <8 x float>@test_int_x86_avx512_mask_movshdup_256(<8 x float> %x0, <8 x float> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_movshdup_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovshdup %ymm0, %ymm1 {%k1} +; CHECK-NEXT: ## ymm1 = ymm0[1,1,3,3,5,5,7,7] +; CHECK-NEXT: vmovshdup %ymm0, %ymm2 {%k1} {z} +; CHECK-NEXT: ## ymm2 = ymm0[1,1,3,3,5,5,7,7] +; CHECK-NEXT: vmovshdup %ymm0, %ymm0 +; CHECK-NEXT: ## ymm0 = ymm0[1,1,3,3,5,5,7,7] +; CHECK-NEXT: vaddps %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: vaddps %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.avx512.mask.movshdup.256(<8 x float> %x0, <8 x float> %x1, i8 %x2) + %res1 = call <8 x float> @llvm.x86.avx512.mask.movshdup.256(<8 x float> %x0, <8 x float> %x1, i8 -1) + %res2 = call <8 x float> @llvm.x86.avx512.mask.movshdup.256(<8 x float> %x0, <8 x float> zeroinitializer, i8 %x2) + %res3 = fadd <8 x float> %res, %res1 + %res4 = fadd <8 x float> %res2, %res3 + ret <8 x float> %res4 +} +declare <2 x double> @llvm.x86.avx512.mask.movddup.128(<2 x double>, <2 x double>, i8) + +define <2 x double>@test_int_x86_avx512_mask_movddup_128(<2 x double> %x0, <2 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_movddup_128: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovddup %xmm0, %xmm1 {%k1} +; CHECK-NEXT: ## xmm1 = xmm0[0,0] +; CHECK-NEXT: vmovddup %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: ## xmm2 = xmm0[0,0] +; CHECK-NEXT: vmovddup %xmm0, %xmm0 +; CHECK-NEXT: ## xmm0 = xmm0[0,0] +; CHECK-NEXT: vaddpd %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: vaddpd %xmm0, %xmm2, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.avx512.mask.movddup.128(<2 x double> %x0, <2 x double> %x1, i8 %x2) + %res1 = call <2 x double> @llvm.x86.avx512.mask.movddup.128(<2 x double> %x0, <2 x double> %x1, i8 -1) + %res2 = call <2 x double> @llvm.x86.avx512.mask.movddup.128(<2 x double> %x0, <2 x double> zeroinitializer, i8 %x2) + %res3 = fadd <2 x double> %res, %res1 + %res4 = fadd <2 x double> %res2, %res3 + ret <2 x double> %res4 +} + +declare <4 x double> @llvm.x86.avx512.mask.movddup.256(<4 x double>, <4 x double>, i8) + +define <4 x double>@test_int_x86_avx512_mask_movddup_256(<4 x double> %x0, <4 x double> %x1, i8 %x2) { +; CHECK-LABEL: test_int_x86_avx512_mask_movddup_256: +; CHECK: ## BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: kmovw %eax, %k1 +; CHECK-NEXT: vmovddup %ymm0, %ymm1 {%k1} +; CHECK-NEXT: ## ymm1 = ymm0[0,0,2,2] +; CHECK-NEXT: vmovddup %ymm0, %ymm2 {%k1} {z} +; CHECK-NEXT: ## ymm2 = ymm0[0,0,2,2] +; CHECK-NEXT: vmovddup %ymm0, %ymm0 +; CHECK-NEXT: ## ymm0 = ymm0[0,0,2,2] +; CHECK-NEXT: vaddpd %ymm0, %ymm1, %ymm0 +; CHECK-NEXT: vaddpd %ymm0, %ymm2, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.avx512.mask.movddup.256(<4 x double> %x0, <4 x double> %x1, i8 %x2) + %res1 = call <4 x double> @llvm.x86.avx512.mask.movddup.256(<4 x double> %x0, <4 x double> %x1, i8 -1) + %res2 = call <4 x double> @llvm.x86.avx512.mask.movddup.256(<4 x double> %x0, <4 x double> zeroinitializer, i8 %x2) + %res3 = fadd <4 x double> %res, %res1 + %res4 = fadd <4 x double> %res2, %res3 + ret <4 x double> %res4 +} + +define <8 x float> @test_rsqrt_ps_256_rr(<8 x float> %a0) { +; CHECK-LABEL: test_rsqrt_ps_256_rr: +; CHECK: vrsqrt14ps %ymm0, %ymm0 + %res = call <8 x float> @llvm.x86.avx512.rsqrt14.ps.256(<8 x float> %a0, <8 x float> zeroinitializer, i8 -1) + ret <8 x float> %res +} + +define <8 x float> @test_rsqrt_ps_256_rrkz(<8 x float> %a0, i8 %mask) { +; CHECK-LABEL: test_rsqrt_ps_256_rrkz: +; CHECK: vrsqrt14ps %ymm0, %ymm0 {%k1} {z} + %res = call <8 x float> @llvm.x86.avx512.rsqrt14.ps.256(<8 x float> %a0, <8 x float> zeroinitializer, i8 %mask) + ret <8 x float> %res +} + +define <8 x float> @test_rsqrt_ps_256_rrk(<8 x float> %a0, <8 x float> %a1, i8 %mask) { +; CHECK-LABEL: test_rsqrt_ps_256_rrk: +; CHECK: vrsqrt14ps %ymm0, %ymm1 {%k1} + %res = call <8 x float> @llvm.x86.avx512.rsqrt14.ps.256(<8 x float> %a0, <8 x float> %a1, i8 %mask) + ret <8 x float> %res +} + +define <4 x float> @test_rsqrt_ps_128_rr(<4 x float> %a0) { +; CHECK-LABEL: test_rsqrt_ps_128_rr: +; CHECK: vrsqrt14ps %xmm0, %xmm0 + %res = call <4 x float> @llvm.x86.avx512.rsqrt14.ps.128(<4 x float> %a0, <4 x float> zeroinitializer, i8 -1) + ret <4 x float> %res +} + +define <4 x float> @test_rsqrt_ps_128_rrkz(<4 x float> %a0, i8 %mask) { +; CHECK-LABEL: test_rsqrt_ps_128_rrkz: +; CHECK: vrsqrt14ps %xmm0, %xmm0 {%k1} {z} + %res = call <4 x float> @llvm.x86.avx512.rsqrt14.ps.128(<4 x float> %a0, <4 x float> zeroinitializer, i8 %mask) + ret <4 x float> %res +} + +define <4 x float> @test_rsqrt_ps_128_rrk(<4 x float> %a0, <4 x float> %a1, i8 %mask) { +; CHECK-LABEL: test_rsqrt_ps_128_rrk: +; CHECK: vrsqrt14ps %xmm0, %xmm1 {%k1} + %res = call <4 x float> @llvm.x86.avx512.rsqrt14.ps.128(<4 x float> %a0, <4 x float> %a1, i8 %mask) + ret <4 x float> %res +} + +declare <8 x float> @llvm.x86.avx512.rsqrt14.ps.256(<8 x float>, <8 x float>, i8) nounwind readnone +declare <4 x float> @llvm.x86.avx512.rsqrt14.ps.128(<4 x float>, <4 x float>, i8) nounwind readnone + +define <8 x float> @test_rcp_ps_256_rr(<8 x float> %a0) { +; CHECK-LABEL: test_rcp_ps_256_rr: +; CHECK: vrcp14ps %ymm0, %ymm0 + %res = call <8 x float> @llvm.x86.avx512.rcp14.ps.256(<8 x float> %a0, <8 x float> zeroinitializer, i8 -1) + ret <8 x float> %res +} + +define <8 x float> @test_rcp_ps_256_rrkz(<8 x float> %a0, i8 %mask) { +; CHECK-LABEL: test_rcp_ps_256_rrkz: +; CHECK: vrcp14ps %ymm0, %ymm0 {%k1} {z} + %res = call <8 x float> @llvm.x86.avx512.rcp14.ps.256(<8 x float> %a0, <8 x float> zeroinitializer, i8 %mask) + ret <8 x float> %res +} + +define <8 x float> @test_rcp_ps_256_rrk(<8 x float> %a0, <8 x float> %a1, i8 %mask) { +; CHECK-LABEL: test_rcp_ps_256_rrk: +; CHECK: vrcp14ps %ymm0, %ymm1 {%k1} + %res = call <8 x float> @llvm.x86.avx512.rcp14.ps.256(<8 x float> %a0, <8 x float> %a1, i8 %mask) + ret <8 x float> %res +} + +define <4 x float> @test_rcp_ps_128_rr(<4 x float> %a0) { +; CHECK-LABEL: test_rcp_ps_128_rr: +; CHECK: vrcp14ps %xmm0, %xmm0 + %res = call <4 x float> @llvm.x86.avx512.rcp14.ps.128(<4 x float> %a0, <4 x float> zeroinitializer, i8 -1) + ret <4 x float> %res +} + +define <4 x float> @test_rcp_ps_128_rrkz(<4 x float> %a0, i8 %mask) { +; CHECK-LABEL: test_rcp_ps_128_rrkz: +; CHECK: vrcp14ps %xmm0, %xmm0 {%k1} {z} + %res = call <4 x float> @llvm.x86.avx512.rcp14.ps.128(<4 x float> %a0, <4 x float> zeroinitializer, i8 %mask) + ret <4 x float> %res +} + +define <4 x float> @test_rcp_ps_128_rrk(<4 x float> %a0, <4 x float> %a1, i8 %mask) { +; CHECK-LABEL: test_rcp_ps_128_rrk: +; CHECK: vrcp14ps %xmm0, %xmm1 {%k1} + %res = call <4 x float> @llvm.x86.avx512.rcp14.ps.128(<4 x float> %a0, <4 x float> %a1, i8 %mask) + ret <4 x float> %res +} + +declare <8 x float> @llvm.x86.avx512.rcp14.ps.256(<8 x float>, <8 x float>, i8) nounwind readnone +declare <4 x float> @llvm.x86.avx512.rcp14.ps.128(<4 x float>, <4 x float>, i8) nounwind readnone + + +define <4 x double> @test_rsqrt_pd_256_rr(<4 x double> %a0) { +; CHECK-LABEL: test_rsqrt_pd_256_rr: +; CHECK: vrsqrt14pd %ymm0, %ymm0 + %res = call <4 x double> @llvm.x86.avx512.rsqrt14.pd.256(<4 x double> %a0, <4 x double> zeroinitializer, i8 -1) + ret <4 x double> %res +} + +define <4 x double> @test_rsqrt_pd_256_rrkz(<4 x double> %a0, i8 %mask) { +; CHECK-LABEL: test_rsqrt_pd_256_rrkz: +; CHECK: vrsqrt14pd %ymm0, %ymm0 {%k1} {z} + %res = call <4 x double> @llvm.x86.avx512.rsqrt14.pd.256(<4 x double> %a0, <4 x double> zeroinitializer, i8 %mask) + ret <4 x double> %res +} + +define <4 x double> @test_rsqrt_pd_256_rrk(<4 x double> %a0, <4 x double> %a1, i8 %mask) { +; CHECK-LABEL: test_rsqrt_pd_256_rrk: +; CHECK: vrsqrt14pd %ymm0, %ymm1 {%k1} + %res = call <4 x double> @llvm.x86.avx512.rsqrt14.pd.256(<4 x double> %a0, <4 x double> %a1, i8 %mask) + ret <4 x double> %res +} + +define <2 x double> @test_rsqrt_pd_128_rr(<2 x double> %a0) { +; CHECK-LABEL: test_rsqrt_pd_128_rr: +; CHECK: vrsqrt14pd %xmm0, %xmm0 + %res = call <2 x double> @llvm.x86.avx512.rsqrt14.pd.128(<2 x double> %a0, <2 x double> zeroinitializer, i8 -1) + ret <2 x double> %res +} + +define <2 x double> @test_rsqrt_pd_128_rrkz(<2 x double> %a0, i8 %mask) { +; CHECK-LABEL: test_rsqrt_pd_128_rrkz: +; CHECK: vrsqrt14pd %xmm0, %xmm0 {%k1} {z} + %res = call <2 x double> @llvm.x86.avx512.rsqrt14.pd.128(<2 x double> %a0, <2 x double> zeroinitializer, i8 %mask) + ret <2 x double> %res +} + +define <2 x double> @test_rsqrt_pd_128_rrk(<2 x double> %a0, <2 x double> %a1, i8 %mask) { +; CHECK-LABEL: test_rsqrt_pd_128_rrk: +; CHECK: vrsqrt14pd %xmm0, %xmm1 {%k1} + %res = call <2 x double> @llvm.x86.avx512.rsqrt14.pd.128(<2 x double> %a0, <2 x double> %a1, i8 %mask) + ret <2 x double> %res +} + +declare <4 x double> @llvm.x86.avx512.rsqrt14.pd.256(<4 x double>, <4 x double>, i8) nounwind readnone +declare <2 x double> @llvm.x86.avx512.rsqrt14.pd.128(<2 x double>, <2 x double>, i8) nounwind readnone + +define <4 x double> @test_rcp_pd_256_rr(<4 x double> %a0) { +; CHECK-LABEL: test_rcp_pd_256_rr: +; CHECK: vrcp14pd %ymm0, %ymm0 + %res = call <4 x double> @llvm.x86.avx512.rcp14.pd.256(<4 x double> %a0, <4 x double> zeroinitializer, i8 -1) + ret <4 x double> %res +} + +define <4 x double> @test_rcp_pd_256_rrkz(<4 x double> %a0, i8 %mask) { +; CHECK-LABEL: test_rcp_pd_256_rrkz: +; CHECK: vrcp14pd %ymm0, %ymm0 {%k1} {z} + %res = call <4 x double> @llvm.x86.avx512.rcp14.pd.256(<4 x double> %a0, <4 x double> zeroinitializer, i8 %mask) + ret <4 x double> %res +} + +define <4 x double> @test_rcp_pd_256_rrk(<4 x double> %a0, <4 x double> %a1, i8 %mask) { +; CHECK-LABEL: test_rcp_pd_256_rrk: +; CHECK: vrcp14pd %ymm0, %ymm1 {%k1} + %res = call <4 x double> @llvm.x86.avx512.rcp14.pd.256(<4 x double> %a0, <4 x double> %a1, i8 %mask) + ret <4 x double> %res +} + +define <2 x double> @test_rcp_pd_128_rr(<2 x double> %a0) { +; CHECK-LABEL: test_rcp_pd_128_rr: +; CHECK: vrcp14pd %xmm0, %xmm0 + %res = call <2 x double> @llvm.x86.avx512.rcp14.pd.128(<2 x double> %a0, <2 x double> zeroinitializer, i8 -1) + ret <2 x double> %res +} + +define <2 x double> @test_rcp_pd_128_rrkz(<2 x double> %a0, i8 %mask) { +; CHECK-LABEL: test_rcp_pd_128_rrkz: +; CHECK: vrcp14pd %xmm0, %xmm0 {%k1} {z} + %res = call <2 x double> @llvm.x86.avx512.rcp14.pd.128(<2 x double> %a0, <2 x double> zeroinitializer, i8 %mask) + ret <2 x double> %res +} + +define <2 x double> @test_rcp_pd_128_rrk(<2 x double> %a0, <2 x double> %a1, i8 %mask) { +; CHECK-LABEL: test_rcp_pd_128_rrk: +; CHECK: vrcp14pd %xmm0, %xmm1 {%k1} + %res = call <2 x double> @llvm.x86.avx512.rcp14.pd.128(<2 x double> %a0, <2 x double> %a1, i8 %mask) + ret <2 x double> %res +} + +declare <4 x double> @llvm.x86.avx512.rcp14.pd.256(<4 x double>, <4 x double>, i8) nounwind readnone +declare <2 x double> @llvm.x86.avx512.rcp14.pd.128(<2 x double>, <2 x double>, i8) nounwind readnone + +define <4 x double> @test_x86_vbroadcast_sd_pd_256(<2 x double> %a0, <4 x double> %a1, i8 %mask ) { +; CHECK-LABEL: test_x86_vbroadcast_sd_pd_256: +; CHECK: kmovw %eax, %k1 +; CHECK-NEXT: vbroadcastsd %xmm0, %ymm1 {%k1} +; CHECK-NEXT: vbroadcastsd %xmm0, %ymm2 {%k1} {z} +; CHECK-NEXT: vbroadcastsd %xmm0, %ymm0 +; CHECK-NEXT: vaddpd %ymm1, %ymm0, %ymm0 + + %res = call <4 x double> @llvm.x86.avx512.mask.broadcast.sd.pd.256(<2 x double> %a0, <4 x double> zeroinitializer, i8 -1) + %res1 = call <4 x double> @llvm.x86.avx512.mask.broadcast.sd.pd.256(<2 x double> %a0, <4 x double> %a1, i8 %mask) + %res2 = call <4 x double> @llvm.x86.avx512.mask.broadcast.sd.pd.256(<2 x double> %a0, <4 x double> zeroinitializer, i8 %mask) + %res3 = fadd <4 x double> %res, %res1 + %res4 = fadd <4 x double> %res2, %res3 + ret <4 x double> %res4 +} +declare <4 x double> @llvm.x86.avx512.mask.broadcast.sd.pd.256(<2 x double>, <4 x double>, i8) nounwind readonly + +define <8 x float> @test_x86_vbroadcast_ss_ps_256(<4 x float> %a0, <8 x float> %a1, i8 %mask ) { +; CHECK-LABEL: test_x86_vbroadcast_ss_ps_256: +; CHECK: kmovw %eax, %k1 +; CHECK-NEXT: vbroadcastss %xmm0, %ymm1 {%k1} +; CHECK-NEXT: vbroadcastss %xmm0, %ymm2 {%k1} {z} +; CHECK-NEXT: vbroadcastss %xmm0, %ymm0 +; CHECK-NEXT: vaddps %ymm1, %ymm0, %ymm0 + + %res = call <8 x float> @llvm.x86.avx512.mask.broadcast.ss.ps.256(<4 x float> %a0, <8 x float> zeroinitializer, i8 -1) + %res1 = call <8 x float> @llvm.x86.avx512.mask.broadcast.ss.ps.256(<4 x float> %a0, <8 x float> %a1, i8 %mask) + %res2 = call <8 x float> @llvm.x86.avx512.mask.broadcast.ss.ps.256(<4 x float> %a0, <8 x float> zeroinitializer, i8 %mask) + %res3 = fadd <8 x float> %res, %res1 + %res4 = fadd <8 x float> %res2, %res3 + ret <8 x float> %res4 +} +declare <8 x float> @llvm.x86.avx512.mask.broadcast.ss.ps.256(<4 x float>, <8 x float>, i8) nounwind readonly + +define <4 x float> @test_x86_vbroadcast_ss_ps_128(<4 x float> %a0, <4 x float> %a1, i8 %mask ) { +; CHECK-LABEL: test_x86_vbroadcast_ss_ps_128: +; CHECK: kmovw %eax, %k1 +; CHECK-NEXT: vbroadcastss %xmm0, %xmm1 {%k1} +; CHECK-NEXT: vbroadcastss %xmm0, %xmm2 {%k1} {z} +; CHECK-NEXT: vbroadcastss %xmm0, %xmm0 +; CHECK-NEXT: vaddps %xmm1, %xmm0, %xmm0 + + %res = call <4 x float> @llvm.x86.avx512.mask.broadcast.ss.ps.128(<4 x float> %a0, <4 x float> zeroinitializer, i8 -1) + %res1 = call <4 x float> @llvm.x86.avx512.mask.broadcast.ss.ps.128(<4 x float> %a0, <4 x float> %a1, i8 %mask) + %res2 = call <4 x float> @llvm.x86.avx512.mask.broadcast.ss.ps.128(<4 x float> %a0, <4 x float> zeroinitializer, i8 %mask) + %res3 = fadd <4 x float> %res, %res1 + %res4 = fadd <4 x float> %res2, %res3 + ret <4 x float> %res4 +} +declare <4 x float> @llvm.x86.avx512.mask.broadcast.ss.ps.128(<4 x float>, <4 x float>, i8) nounwind readonly + + +declare <8 x float> @llvm.x86.avx512.mask.broadcastf32x4.256(<4 x float>, <8 x float>, i8) + +define <8 x float>@test_int_x86_avx512_mask_broadcastf32x4_256(<4 x float> %x0, <8 x float> %x2, i8 %mask) { +; CHECK-LABEL: test_int_x86_avx512_mask_broadcastf32x4_256: +; CHECK: kmovw %eax, %k1 +; CHECK: vshuff32x4 $0, %ymm0, %ymm0, %ymm2 {%k1} {z} +; CHECK: vshuff32x4 $0, %ymm0, %ymm0, %ymm1 {%k1} +; CHECK: vshuff32x4 $0, %ymm0, %ymm0, %ymm0 +; CHECK: vaddps %ymm1, %ymm0, %ymm0 +; CHECK: vaddps %ymm0, %ymm2, %ymm0 + + %res1 = call <8 x float> @llvm.x86.avx512.mask.broadcastf32x4.256(<4 x float> %x0, <8 x float> %x2, i8 -1) + %res2 = call <8 x float> @llvm.x86.avx512.mask.broadcastf32x4.256(<4 x float> %x0, <8 x float> %x2, i8 %mask) + %res3 = call <8 x float> @llvm.x86.avx512.mask.broadcastf32x4.256(<4 x float> %x0, <8 x float> zeroinitializer, i8 %mask) + %res4 = fadd <8 x float> %res1, %res2 + %res5 = fadd <8 x float> %res3, %res4 + ret <8 x float> %res5 +} + +declare <8 x i32> @llvm.x86.avx512.mask.broadcasti32x4.256(<4 x i32>, <8 x i32>, i8) + +define <8 x i32>@test_int_x86_avx512_mask_broadcasti32x4_256(<4 x i32> %x0, <8 x i32> %x2, i8 %mask) { +; CHECK-LABEL: test_int_x86_avx512_mask_broadcasti32x4_256: +; CHECK: kmovw %eax, %k1 +; CHECK: vshufi32x4 $0, %ymm0, %ymm0, %ymm2 {%k1} {z} +; CHECK: vshufi32x4 $0, %ymm0, %ymm0, %ymm1 {%k1} +; CHECK: vshufi32x4 $0, %ymm0, %ymm0, %ymm0 +; CHECK: vpaddd %ymm1, %ymm0, %ymm0 +; CHECK: vpaddd %ymm0, %ymm2, %ymm0 + + %res1 = call <8 x i32> @llvm.x86.avx512.mask.broadcasti32x4.256(<4 x i32> %x0, <8 x i32> %x2, i8 -1) + %res2 = call <8 x i32> @llvm.x86.avx512.mask.broadcasti32x4.256(<4 x i32> %x0, <8 x i32> %x2, i8 %mask) + %res3 = call <8 x i32> @llvm.x86.avx512.mask.broadcasti32x4.256(<4 x i32> %x0, <8 x i32> zeroinitializer, i8 %mask) + %res4 = add <8 x i32> %res1, %res2 + %res5 = add <8 x i32> %res3, %res4 + ret <8 x i32> %res5 +} diff --git a/test/CodeGen/X86/bit-piece-comment.ll b/test/CodeGen/X86/bit-piece-comment.ll new file mode 100644 index 000000000000..6ce858b11dcf --- /dev/null +++ b/test/CodeGen/X86/bit-piece-comment.ll @@ -0,0 +1,64 @@ +; RUN: llc -filetype=asm < %s +; +; We check that we don't crash when printing assembly comments that include +; a DW_OP_bit_piece +; +; Regenerate from +; void fn1() { +; struct { +; int dword[2]; +; } u; +; u.dword[1] = 0; +; }; +; via clang++ -g -fno-integrated-as -Os + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + + +%struct.anon = type { [2 x i32] } + +; Function Attrs: norecurse nounwind optsize readnone uwtable +define void @_Z3fn1v() #0 !dbg !4 { +entry: + tail call void @llvm.dbg.declare(metadata %struct.anon* undef, metadata !8, metadata !19), !dbg !20 + tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !8, metadata !21), !dbg !20 + ret void, !dbg !22 +} + +; Function Attrs: nounwind readnone +declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 + +; Function Attrs: nounwind readnone +declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1 + +attributes #0 = { norecurse nounwind optsize readnone uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { nounwind readnone } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!16, !17} +!llvm.ident = !{!18} + +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 256088) (llvm/trunk 256097)", isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !2, subprograms: !3) +!1 = !DIFile(filename: "test.cpp", directory: "/mnt/extra") +!2 = !{} +!3 = !{!4} +!4 = distinct !DISubprogram(name: "fn1", linkageName: "_Z3fn1v", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, variables: !7) +!5 = !DISubroutineType(types: !6) +!6 = !{null} +!7 = !{!8} +!8 = !DILocalVariable(name: "u", scope: !4, file: !1, line: 4, type: !9) +!9 = !DICompositeType(tag: DW_TAG_structure_type, scope: !4, file: !1, line: 2, size: 64, align: 32, elements: !10) +!10 = !{!11} +!11 = !DIDerivedType(tag: DW_TAG_member, name: "dword", scope: !9, file: !1, line: 3, baseType: !12, size: 64, align: 32) +!12 = !DICompositeType(tag: DW_TAG_array_type, baseType: !13, size: 64, align: 32, elements: !14) +!13 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) +!14 = !{!15} +!15 = !DISubrange(count: 2) +!16 = !{i32 2, !"Dwarf Version", i32 4} +!17 = !{i32 2, !"Debug Info Version", i32 3} +!18 = !{!"clang version 3.8.0 (trunk 256088) (llvm/trunk 256097)"} +!19 = !DIExpression() +!20 = !DILocation(line: 4, column: 5, scope: !4) +!21 = !DIExpression(DW_OP_bit_piece, 32, 32) +!22 = !DILocation(line: 6, column: 1, scope: !4) diff --git a/test/CodeGen/X86/bitreverse.ll b/test/CodeGen/X86/bitreverse.ll new file mode 100644 index 000000000000..e3bc8ace38ab --- /dev/null +++ b/test/CodeGen/X86/bitreverse.ll @@ -0,0 +1,22 @@ +; RUN: llc -march=x86 %s -o - | FileCheck %s + +; These tests just check that the plumbing is in place for @llvm.bitreverse. The +; actual output is massive at the moment as llvm.bitreverse is not yet legal. + +declare <2 x i16> @llvm.bitreverse.v2i16(<2 x i16>) readnone + +define <2 x i16> @f(<2 x i16> %a) { +; CHECK-LABEL: f: +; CHECK: shll + %b = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> %a) + ret <2 x i16> %b +} + +declare i8 @llvm.bitreverse.i8(i8) readnone + +define i8 @g(i8 %a) { +; CHECK-LABEL: g: +; CHECK: shlb + %b = call i8 @llvm.bitreverse.i8(i8 %a) + ret i8 %b +} diff --git a/test/CodeGen/X86/branchfolding-catchpads.ll b/test/CodeGen/X86/branchfolding-catchpads.ll new file mode 100644 index 000000000000..0468b3c314f6 --- /dev/null +++ b/test/CodeGen/X86/branchfolding-catchpads.ll @@ -0,0 +1,95 @@ +; RUN: llc -mtriple=x86_64-pc-windows-msvc < %s | FileCheck %s + +declare i32 @__CxxFrameHandler3(...) + +declare void @throw() +declare i16 @f() + +define i16 @test1(i16 %a, i8* %b) personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) { +entry: + %cmp = icmp eq i16 %a, 10 + br i1 %cmp, label %if.then, label %if.else + +if.then: + %call1 = invoke i16 @f() + to label %cleanup unwind label %catch.dispatch + +if.else: + %call2 = invoke i16 @f() + to label %cleanup unwind label %catch.dispatch + +catch.dispatch: + %cs = catchswitch within none [ label %catch, label %catch.2 ] unwind to caller + +catch: + catchpad within %cs [i8* null, i32 8, i8* null] + call void @throw() noreturn + br label %unreachable + +catch.2: + catchpad within %cs [i8* null, i32 64, i8* null] + store i8 1, i8* %b + call void @throw() noreturn + br label %unreachable + +cleanup: + %retval = phi i16 [ %call1, %if.then ], [ %call2, %if.else ] + ret i16 %retval + +unreachable: + unreachable +} + +; This test verifies the case where two funclet blocks meet the old criteria +; to be placed at the end. The order of the blocks is not important for the +; purposes of this test. The failure mode is an infinite loop during +; compilation. +; +; CHECK-LABEL: .def test1; + +define i16 @test2(i16 %a, i8* %b) personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) { +entry: + %cmp = icmp eq i16 %a, 10 + br i1 %cmp, label %if.then, label %if.else + +if.then: + %call1 = invoke i16 @f() + to label %cleanup unwind label %catch.dispatch + +if.else: + %call2 = invoke i16 @f() + to label %cleanup unwind label %catch.dispatch + +catch.dispatch: + %cs = catchswitch within none [ label %catch, label %catch.2, label %catch.3 ] unwind to caller + +catch: + catchpad within %cs [i8* null, i32 8, i8* null] + call void @throw() noreturn + br label %unreachable + +catch.2: + %c2 = catchpad within %cs [i8* null, i32 32, i8* null] + store i8 1, i8* %b + catchret from %c2 to label %cleanup + +catch.3: + %c3 = catchpad within %cs [i8* null, i32 64, i8* null] + store i8 2, i8* %b + catchret from %c3 to label %cleanup + +cleanup: + %retval = phi i16 [ %call1, %if.then ], [ %call2, %if.else ], [ -1, %catch.2 ], [ -1, %catch.3 ] + ret i16 %retval + +unreachable: + unreachable +} + +; This test verifies the case where three funclet blocks all meet the old +; criteria to be placed at the end. The order of the blocks is not important +; for the purposes of this test. The failure mode is an infinite loop during +; compilation. +; +; CHECK-LABEL: .def test2; + diff --git a/test/CodeGen/X86/buildvec-insertvec.ll b/test/CodeGen/X86/buildvec-insertvec.ll index 73dbe1f650a1..fd7290d58179 100644 --- a/test/CodeGen/X86/buildvec-insertvec.ll +++ b/test/CodeGen/X86/buildvec-insertvec.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s define void @foo(<3 x float> %in, <4 x i8>* nocapture %out) nounwind { diff --git a/test/CodeGen/X86/catchpad-realign-savexmm.ll b/test/CodeGen/X86/catchpad-realign-savexmm.ll new file mode 100644 index 000000000000..1160101792ff --- /dev/null +++ b/test/CodeGen/X86/catchpad-realign-savexmm.ll @@ -0,0 +1,53 @@ +; RUN: llc -mtriple=x86_64-pc-windows-msvc -verify-machineinstrs < %s | FileCheck %s + +; We should store -2 into UnwindHelp in a slot immediately after the last XMM +; CSR save. + +declare void @g() +declare i32 @__CxxFrameHandler3(...) + +@fp_global = global double 0.0 + +define void @f() personality i32 (...)* @__CxxFrameHandler3 { + %v = load double, double* @fp_global + call void @g() + %v1 = fadd double %v, 1.0 + store double %v1, double* @fp_global + invoke void @g() + to label %return unwind label %catch.dispatch + +return: + ret void + +catch.dispatch: + %cs1 = catchswitch within none [label %catch] unwind to caller + +catch: + %p = catchpad within %cs1 [i8* null, i32 64, i8* null] + catchret from %p to label %return +} + +; CHECK: f: # @f +; CHECK: pushq %rbp +; CHECK: .seh_pushreg 5 +; CHECK: subq $64, %rsp +; CHECK: .seh_stackalloc 64 +; CHECK: leaq 64(%rsp), %rbp +; CHECK: .seh_setframe 5, 64 +; CHECK: movaps %xmm6, -16(%rbp) # 16-byte Spill +; CHECK: .seh_savexmm 6, 48 +; CHECK: .seh_endprologue +; CHECK: movq $-2, -24(%rbp) +; CHECK: movsd fp_global(%rip), %xmm6 # xmm6 = mem[0],zero +; CHECK: callq g +; CHECK: addsd __real@3ff0000000000000(%rip), %xmm6 +; CHECK: movsd %xmm6, fp_global(%rip) +; CHECK: .Ltmp{{.*}} +; CHECK: callq g +; CHECK: .Ltmp{{.*}} +; CHECK: .LBB{{.*}} # Block address taken +; CHECK: movaps -16(%rbp), %xmm6 +; CHECK: addq $64, %rsp +; CHECK: popq %rbp +; CHECK: retq +; CHECK: .seh_handlerdata diff --git a/test/CodeGen/X86/catchpad-regmask.ll b/test/CodeGen/X86/catchpad-regmask.ll new file mode 100644 index 000000000000..0d436f6eb595 --- /dev/null +++ b/test/CodeGen/X86/catchpad-regmask.ll @@ -0,0 +1,144 @@ +; RUN: llc < %s | FileCheck %s + +; Based on this code: +; +; extern "C" int array[4]; +; extern "C" void global_array(int idx1, int idx2, int idx3) { +; try { +; array[idx1] = 111; +; throw; +; } catch (...) { +; array[idx2] = 222; +; } +; array[idx3] = 333; +; } +; extern "C" __declspec(dllimport) int imported; +; extern "C" void access_imported() { +; try { +; imported = 111; +; throw; +; } catch (...) { +; imported = 222; +; } +; imported = 333; +; } + +target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc18.0.0" + +%eh.ThrowInfo = type { i32, i32, i32, i32 } + +@array = external global [4 x i32], align 16 +@imported = external dllimport global i32, align 4 + +; Function Attrs: uwtable +define void @global_array(i32 %idx1, i32 %idx2, i32 %idx3) #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) { +entry: + %idxprom = sext i32 %idx1 to i64 + %arrayidx = getelementptr inbounds [4 x i32], [4 x i32]* @array, i64 0, i64 %idxprom + store i32 111, i32* %arrayidx, align 4, !tbaa !2 + invoke void @_CxxThrowException(i8* null, %eh.ThrowInfo* null) #1 + to label %unreachable unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %cs1 = catchswitch within none [label %catch] unwind to caller + +catch: ; preds = %catch.dispatch + %0 = catchpad within %cs1 [i8* null, i32 64, i8* null] + %idxprom1 = sext i32 %idx2 to i64 + %arrayidx2 = getelementptr inbounds [4 x i32], [4 x i32]* @array, i64 0, i64 %idxprom1 + store i32 222, i32* %arrayidx2, align 4, !tbaa !2 + catchret from %0 to label %try.cont + +try.cont: ; preds = %catch + %idxprom3 = sext i32 %idx3 to i64 + %arrayidx4 = getelementptr inbounds [4 x i32], [4 x i32]* @array, i64 0, i64 %idxprom3 + store i32 333, i32* %arrayidx4, align 4, !tbaa !2 + ret void + +unreachable: ; preds = %entry + unreachable +} + +; CHECK-LABEL: global_array: # @global_array +; CHECK: pushq %rbp +; First array access +; CHECK: movslq %ecx, %[[idx:[^ ]*]] +; CHECK: leaq array(%rip), %[[base:[^ ]*]] +; CHECK: movl $111, (%[[base]],%[[idx]],4) +; Might throw an exception and return to below... +; CHECK: callq _CxxThrowException +; Third array access must remat the address of array +; CHECK: movslq {{.*}}, %[[idx:[^ ]*]] +; CHECK: leaq array(%rip), %[[base:[^ ]*]] +; CHECK: movl $333, (%[[base]],%[[idx]],4) +; CHECK: popq %rbp +; CHECK: retq + +; CHECK: "?catch$2@?0?global_array@4HA": +; CHECK: pushq %rbp +; CHECK: movslq {{.*}}, %[[idx:[^ ]*]] +; CHECK: leaq array(%rip), %[[base:[^ ]*]] +; CHECK: movl $222, (%[[base]],%[[idx]],4) +; CHECK: popq %rbp +; CHECK: retq # CATCHRET + +declare void @_CxxThrowException(i8*, %eh.ThrowInfo*) + +declare i32 @__CxxFrameHandler3(...) + +; Function Attrs: uwtable +define void @access_imported() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) { +entry: + store i32 111, i32* @imported, align 4, !tbaa !2 + invoke void @_CxxThrowException(i8* null, %eh.ThrowInfo* null) #1 + to label %unreachable unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %cs1 = catchswitch within none [label %catch] unwind to caller + +catch: ; preds = %catch.dispatch + %0 = catchpad within %cs1 [i8* null, i32 64, i8* null] + store i32 222, i32* @imported, align 4, !tbaa !2 + catchret from %0 to label %try.cont + +try.cont: ; preds = %catch + store i32 333, i32* @imported, align 4, !tbaa !2 + ret void + +unreachable: ; preds = %entry + unreachable +} + +; CHECK-LABEL: access_imported: # @access_imported +; CHECK: pushq %rbp +; CHECK: movq __imp_imported(%rip), %[[base:[^ ]*]] +; CHECK: movl $111, (%[[base]]) +; Might throw an exception and return to below... +; CHECK: callq _CxxThrowException +; Third access must reload the address of imported +; CHECK: movq __imp_imported(%rip), %[[base:[^ ]*]] +; CHECK: movl $333, (%[[base]]) +; CHECK: popq %rbp +; CHECK: retq + +; CHECK: "?catch$2@?0?access_imported@4HA": +; CHECK: pushq %rbp +; CHECK: movq __imp_imported(%rip), %[[base:[^ ]*]] +; CHECK: movl $222, (%[[base]]) +; CHECK: popq %rbp +; CHECK: retq # CATCHRET + + +attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { noreturn } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"PIC Level", i32 2} +!1 = !{!"clang version 3.8.0 "} +!2 = !{!3, !3, i64 0} +!3 = !{!"int", !4, i64 0} +!4 = !{!"omnipotent char", !5, i64 0} +!5 = !{!"Simple C/C++ TBAA"} diff --git a/test/CodeGen/X86/catchpad-weight.ll b/test/CodeGen/X86/catchpad-weight.ll new file mode 100644 index 000000000000..60939bc6b03e --- /dev/null +++ b/test/CodeGen/X86/catchpad-weight.ll @@ -0,0 +1,82 @@ +; RUN: llc -march=x86-64 -print-machineinstrs=expand-isel-pseudos %s -o /dev/null 2>&1 | FileCheck %s + +; Check if the edge weight to the catchpad is calculated correctly. + +; CHECK: Successors according to CFG: BB#2(0x7ffff100 / 0x80000000 = 100.00%) BB#1(0x00000800 / 0x80000000 = 0.00%) BB#3(0x00000400 / 0x80000000 = 0.00%) BB#4(0x00000200 / 0x80000000 = 0.00%) BB#5(0x00000100 / 0x80000000 = 0.00%) + +target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64--windows-msvc18.0.0" + +%rtti.TypeDescriptor7 = type { i8**, i8*, [8 x i8] } +%struct.HasDtor = type { i8 } + +$"\01??_R0?AUA@@@8" = comdat any + +$"\01??_R0?AUB@@@8" = comdat any + +$"\01??_R0?AUC@@@8" = comdat any + +@"\01??_7type_info@@6B@" = external constant i8* +@"\01??_R0?AUA@@@8" = linkonce_odr global %rtti.TypeDescriptor7 { i8** @"\01??_7type_info@@6B@", i8* null, [8 x i8] c".?AUA@@\00" }, comdat +@"\01??_R0?AUB@@@8" = linkonce_odr global %rtti.TypeDescriptor7 { i8** @"\01??_7type_info@@6B@", i8* null, [8 x i8] c".?AUB@@\00" }, comdat +@"\01??_R0?AUC@@@8" = linkonce_odr global %rtti.TypeDescriptor7 { i8** @"\01??_7type_info@@6B@", i8* null, [8 x i8] c".?AUC@@\00" }, comdat + +; Function Attrs: uwtable +define i32 @main() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) { +entry: + %o = alloca %struct.HasDtor, align 1 + %0 = getelementptr inbounds %struct.HasDtor, %struct.HasDtor* %o, i64 0, i32 0 + call void @llvm.lifetime.start(i64 1, i8* %0) #4 + invoke void @"\01?may_throw@@YAXXZ"() + to label %try.cont unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %cs1 = catchswitch within none [label %catch.5] unwind label %catch.dispatch.1 + +catch.5: ; preds = %catch.dispatch + %1 = catchpad within %cs1 [%rtti.TypeDescriptor7* @"\01??_R0?AUA@@@8", i32 0, i8* null] + catchret from %1 to label %try.cont + +try.cont: ; preds = %entry, %catch, %catch.3, %catch.5 + call void @"\01??1HasDtor@@QEAA@XZ"(%struct.HasDtor* nonnull %o) #4 + call void @llvm.lifetime.end(i64 1, i8* %0) #4 + ret i32 0 + +catch.dispatch.1: ; preds = %catch.dispatch + %cs2 = catchswitch within none [label %catch.3] unwind label %catch.dispatch.2 + +catch.3: ; preds = %catch.dispatch.1 + %2 = catchpad within %cs2 [%rtti.TypeDescriptor7* @"\01??_R0?AUB@@@8", i32 0, i8* null] + catchret from %2 to label %try.cont + +catch.dispatch.2: ; preds = %catch.dispatch.1 + %cs3 = catchswitch within none [label %catch] unwind label %ehcleanup + +catch: ; preds = %catch.dispatch.2 + %3 = catchpad within %cs3 [%rtti.TypeDescriptor7* @"\01??_R0?AUC@@@8", i32 0, i8* null] + catchret from %3 to label %try.cont + +ehcleanup: ; preds = %catchendblock + %4 = cleanuppad within none [] + call void @"\01??1HasDtor@@QEAA@XZ"(%struct.HasDtor* nonnull %o) #4 [ "funclet"(token %4) ] + cleanupret from %4 unwind to caller +} + +; Function Attrs: nounwind argmemonly +declare void @llvm.lifetime.start(i64, i8* nocapture) #1 + +declare void @"\01?may_throw@@YAXXZ"() #2 + +declare i32 @__CxxFrameHandler3(...) + +; Function Attrs: nounwind +declare void @"\01??1HasDtor@@QEAA@XZ"(%struct.HasDtor*) #3 + +; Function Attrs: nounwind argmemonly +declare void @llvm.lifetime.end(i64, i8* nocapture) #1 + +attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { nounwind argmemonly } +attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #3 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #4 = { nounwind } diff --git a/test/CodeGen/X86/catchret-empty-fallthrough.ll b/test/CodeGen/X86/catchret-empty-fallthrough.ll new file mode 100644 index 000000000000..7ad103303171 --- /dev/null +++ b/test/CodeGen/X86/catchret-empty-fallthrough.ll @@ -0,0 +1,53 @@ +; RUN: llc -verify-machineinstrs < %s | FileCheck %s + +; BranchFolding used to remove our empty landingpad block, which is +; undesirable. + +target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc18.0.0" + +declare i32 @__C_specific_handler(...) + +declare void @bar() + +define void @foo(i1 %cond) personality i32 (...)* @__C_specific_handler { +entry: + br i1 %cond, label %return, label %try + +try: ; preds = %entry + invoke void @bar() + to label %fallthrough unwind label %dispatch + +dispatch: ; preds = %try + %cs1 = catchswitch within none [label %catch] unwind to caller + +catch: ; preds = %dispatch + %0 = catchpad within %cs1 [i8* null] + catchret from %0 to label %return + +fallthrough: ; preds = %try + unreachable + +return: ; preds = %catch, %entry + ret void +} + +; CHECK-LABEL: foo: # @foo +; CHECK: testb $1, %cl +; CHECK: je .LBB0_[[try:[0-9]+]] +; CHECK: .LBB0_[[return:[0-9]+]]: +; CHECK: retq +; CHECK: .LBB0_[[try]]: +; CHECK: .Ltmp0: +; CHECK: callq bar +; CHECK: .Ltmp1: +; CHECK: .LBB0_[[catch:[0-9]+]]: + +; CHECK: .seh_handlerdata +; CHECK-NEXT: .Lfoo$parent_frame_offset = 32 +; CHECK-NEXT: .long (.Llsda_end0-.Llsda_begin0)/16 +; CHECK-NEXT: .Llsda_begin0: +; CHECK-NEXT: .long .Ltmp0@IMGREL+1 +; CHECK-NEXT: .long .Ltmp1@IMGREL+1 +; CHECK-NEXT: .long 1 +; CHECK-NEXT: .long .LBB0_[[catch]]@IMGREL diff --git a/test/CodeGen/X86/catchret-fallthrough.ll b/test/CodeGen/X86/catchret-fallthrough.ll new file mode 100644 index 000000000000..6a94b290e823 --- /dev/null +++ b/test/CodeGen/X86/catchret-fallthrough.ll @@ -0,0 +1,42 @@ +; RUN: llc -verify-machineinstrs < %s | FileCheck %s + +; We used to have an issue where we inserted an MBB between invoke.cont.3 and +; its fallthrough target of ret void. + +target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32" +target triple = "i386-pc-windows-msvc18.0.0" + +@some_global = global i32 0 + +declare i32 @__CxxFrameHandler3(...) + +declare void @g() + +define void @f() personality i32 (...)* @__CxxFrameHandler3 { +entry: + invoke void @g() + to label %invoke.cont.3 unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %cs1 = catchswitch within none [label %catch] unwind to caller + +catch: ; preds = %catch.dispatch + %0 = catchpad within %cs1 [i8* null, i32 64, i8* null] + catchret from %0 to label %nrvo.skipdtor + +invoke.cont.3: ; preds = %entry + store i32 123, i32* @some_global + br label %nrvo.skipdtor + +nrvo.skipdtor: ; preds = %invoke.cont.3, %invoke.cont.4 + ret void +} + +; CHECK-LABEL: _f: # @f +; CHECK: calll _g +; CHECK: movl $123, _some_global +; CHECK-NOT: jmp +; CHECK-NOT: movl {{.*}}, %esp +; CHECK: retl +; CHECK: addl $12, %ebp +; CHECK: jmp LBB0_{{.*}} diff --git a/test/CodeGen/X86/cleanuppad-inalloca.ll b/test/CodeGen/X86/cleanuppad-inalloca.ll new file mode 100644 index 000000000000..2e34ada52e6b --- /dev/null +++ b/test/CodeGen/X86/cleanuppad-inalloca.ll @@ -0,0 +1,68 @@ +; RUN: llc < %s | FileCheck %s + +; Based on this C++: +; struct A { +; int x; +; A(); +; A(const A &a); +; ~A(); +; }; +; extern "C" void takes_two(A a1, A a2); +; extern "C" void passes_two() { takes_two(A(), A()); } + +target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32" +target triple = "i686--windows-msvc" + +%struct.A = type { i32 } + +define void @passes_two() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) { +entry: + %argmem = alloca inalloca <{ %struct.A, %struct.A }>, align 4 + %0 = getelementptr inbounds <{ %struct.A, %struct.A }>, <{ %struct.A, %struct.A }>* %argmem, i32 0, i32 1 + %call = call x86_thiscallcc %struct.A* @"\01??0A@@QAE@XZ"(%struct.A* %0) + %1 = getelementptr inbounds <{ %struct.A, %struct.A }>, <{ %struct.A, %struct.A }>* %argmem, i32 0, i32 0 + %call1 = invoke x86_thiscallcc %struct.A* @"\01??0A@@QAE@XZ"(%struct.A* %1) + to label %invoke.cont unwind label %ehcleanup + +invoke.cont: ; preds = %entry + call void @takes_two(<{ %struct.A, %struct.A }>* inalloca nonnull %argmem) + ret void + +ehcleanup: ; preds = %entry + %2 = cleanuppad within none [] + call x86_thiscallcc void @"\01??1A@@QAE@XZ"(%struct.A* %0) [ "funclet"(token %2) ] + cleanupret from %2 unwind to caller +} + +; CHECK: _passes_two: +; CHECK: pushl %ebp +; CHECK: movl %esp, %ebp +; CHECK: subl ${{[0-9]+}}, %esp +; CHECK: movl $8, %eax +; CHECK: calll __chkstk +; CHECK: calll "??0A@@QAE@XZ" +; CHECK: calll "??0A@@QAE@XZ" +; CHECK: calll _takes_two +; ESP must be restored via EBP due to "dynamic" alloca. +; CHECK: leal -{{[0-9]+}}(%ebp), %esp +; CHECK: popl %ebp +; CHECK: retl + +; CHECK: "?dtor$2@?0?passes_two@4HA": +; CHECK: pushl %ebp +; CHECK: subl $8, %esp +; CHECK: addl $12, %ebp +; CHECK: {{movl|leal}} -{{[0-9]+}}(%ebp), %ecx +; CHECK: calll "??1A@@QAE@XZ" +; CHECK: addl $8, %esp +; CHECK: retl + +declare void @takes_two(<{ %struct.A, %struct.A }>* inalloca) #0 + +declare x86_thiscallcc %struct.A* @"\01??0A@@QAE@XZ"(%struct.A* returned) #0 + +declare i32 @__CxxFrameHandler3(...) + +declare x86_thiscallcc void @"\01??1A@@QAE@XZ"(%struct.A*) #0 + +attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } diff --git a/test/CodeGen/X86/cleanuppad-large-codemodel.ll b/test/CodeGen/X86/cleanuppad-large-codemodel.ll new file mode 100644 index 000000000000..8ffb97d8dd68 --- /dev/null +++ b/test/CodeGen/X86/cleanuppad-large-codemodel.ll @@ -0,0 +1,27 @@ +; RUN: llc -mtriple=x86_64-pc-windows-msvc -code-model=large -o - < %s | FileCheck %s + +declare i32 @__CxxFrameHandler3(...) + +declare void @bar() + +define void @foo() personality i32 (...)* @__CxxFrameHandler3 { +entry: + invoke void @bar() + to label %exit unwind label %cleanup +cleanup: + %c = cleanuppad within none [] + call void @bar() [ "funclet"(token %c) ] + cleanupret from %c unwind to caller +exit: + ret void +} + +; CHECK: foo: # @foo +; CHECK: movabsq $bar, %[[reg:[^ ]*]] +; CHECK: callq *%[[reg]] +; CHECK: retq + +; CHECK: "?dtor$2@?0?foo@4HA": +; CHECK: movabsq $bar, %[[reg:[^ ]*]] +; CHECK: callq *%[[reg]] +; CHECK: retq # CLEANUPRET diff --git a/test/CodeGen/X86/cleanuppad-realign.ll b/test/CodeGen/X86/cleanuppad-realign.ll new file mode 100644 index 000000000000..5a565cc1570f --- /dev/null +++ b/test/CodeGen/X86/cleanuppad-realign.ll @@ -0,0 +1,78 @@ +; RUN: llc -mtriple=i686-pc-windows-msvc < %s | FileCheck --check-prefix=X86 %s +; RUN: llc -mtriple=x86_64-pc-windows-msvc < %s | FileCheck --check-prefix=X64 %s + +declare i32 @__CxxFrameHandler3(...) +declare void @Dtor(i64* %o) +declare void @f(i32) + +define void @realigned_cleanup() personality i32 (...)* @__CxxFrameHandler3 { +entry: + ; Overalign %o to cause stack realignment. + %o = alloca i64, align 32 + invoke void @f(i32 1) + to label %invoke.cont unwind label %ehcleanup + +invoke.cont: ; preds = %entry + call void @Dtor(i64* %o) + ret void + +ehcleanup: ; preds = %entry + %0 = cleanuppad within none [] + call void @Dtor(i64* %o) [ "funclet"(token %0) ] + cleanupret from %0 unwind to caller +} + +; X86-LABEL: _realigned_cleanup: # @realigned_cleanup +; X86: pushl %ebp +; X86: movl %esp, %ebp +; X86: pushl %ebx +; X86: pushl %edi +; X86: pushl %esi +; X86: andl $-32, %esp +; X86: subl $96, %esp +; X86: movl %esp, %esi +; EBP will reload from this offset. +; X86: movl %ebp, 28(%esi) +; The last EH reg field is the state number, so dtor adjust is this +4. +; X86: movl $-1, 72(%esi) + +; X86-LABEL: "?dtor$2@?0?realigned_cleanup@4HA": +; X86: pushl %ebp +; X86: leal -76(%ebp), %esi +; X86: movl 28(%esi), %ebp +; We used to have a bug where we clobbered ESI after the prologue. +; X86-NOT: movl {{.*}}, %esi +; X86: popl %ebp +; X86: retl # CLEANUPRET + +; X64-LABEL: realigned_cleanup: # @realigned_cleanup +; X64: pushq %rbp +; X64: .seh_pushreg 5 +; X64: pushq %rbx +; X64: .seh_pushreg 3 +; X64: subq $104, %rsp +; X64: .seh_stackalloc 104 +; X64: leaq 96(%rsp), %rbp +; X64: .seh_setframe 5, 96 +; X64: .seh_endprologue +; X64: andq $-32, %rsp +; X64: movq %rsp, %rbx +; RBP will reload from this offset. +; X64: movq %rbp, 56(%rbx) +; X64: movq $-2, (%rbp) + +; X64-LABEL: "?dtor$2@?0?realigned_cleanup@4HA": +; X64: movq %rdx, 16(%rsp) +; X64: pushq %rbp +; X64: .seh_pushreg 5 +; X64: pushq %rbx +; X64: .seh_pushreg 3 +; X64: subq $40, %rsp +; X64: .seh_stackalloc 40 +; X64: leaq 96(%rdx), %rbp +; X64: .seh_endprologue +; X64: andq $-32, %rdx +; X64: movq %rdx, %rbx +; X64-NOT: mov{{.*}}, %rbx +; X64: popq %rbp +; X64: retq # CLEANUPRET diff --git a/test/CodeGen/X86/clz.ll b/test/CodeGen/X86/clz.ll index 6a6f5256f44d..4a094480c931 100644 --- a/test/CodeGen/X86/clz.ll +++ b/test/CodeGen/X86/clz.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -march=x86-64 -mcpu=yonah | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s declare i8 @llvm.cttz.i8(i8, i1) declare i16 @llvm.cttz.i16(i16, i1) @@ -10,131 +10,151 @@ declare i32 @llvm.ctlz.i32(i32, i1) declare i64 @llvm.ctlz.i64(i64, i1) define i8 @cttz_i8(i8 %x) { +; CHECK-LABEL: cttz_i8: +; CHECK: # BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: bsfl %eax, %eax +; CHECK-NEXT: retq %tmp = call i8 @llvm.cttz.i8( i8 %x, i1 true ) ret i8 %tmp -; CHECK-LABEL: cttz_i8: -; CHECK: bsfl -; CHECK-NOT: cmov -; CHECK: ret } define i16 @cttz_i16(i16 %x) { +; CHECK-LABEL: cttz_i16: +; CHECK: # BB#0: +; CHECK-NEXT: bsfw %di, %ax +; CHECK-NEXT: retq %tmp = call i16 @llvm.cttz.i16( i16 %x, i1 true ) ret i16 %tmp -; CHECK-LABEL: cttz_i16: -; CHECK: bsfw -; CHECK-NOT: cmov -; CHECK: ret } define i32 @cttz_i32(i32 %x) { +; CHECK-LABEL: cttz_i32: +; CHECK: # BB#0: +; CHECK-NEXT: bsfl %edi, %eax +; CHECK-NEXT: retq %tmp = call i32 @llvm.cttz.i32( i32 %x, i1 true ) ret i32 %tmp -; CHECK-LABEL: cttz_i32: -; CHECK: bsfl -; CHECK-NOT: cmov -; CHECK: ret } define i64 @cttz_i64(i64 %x) { +; CHECK-LABEL: cttz_i64: +; CHECK: # BB#0: +; CHECK-NEXT: bsfq %rdi, %rax +; CHECK-NEXT: retq %tmp = call i64 @llvm.cttz.i64( i64 %x, i1 true ) ret i64 %tmp -; CHECK-LABEL: cttz_i64: -; CHECK: bsfq -; CHECK-NOT: cmov -; CHECK: ret } define i8 @ctlz_i8(i8 %x) { -entry: +; CHECK-LABEL: ctlz_i8: +; CHECK: # BB#0: +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: bsrl %eax, %eax +; CHECK-NEXT: xorl $7, %eax +; CHECK-NEXT: retq %tmp2 = call i8 @llvm.ctlz.i8( i8 %x, i1 true ) ret i8 %tmp2 -; CHECK-LABEL: ctlz_i8: -; CHECK: bsrl -; CHECK-NOT: cmov -; CHECK: xorl $7, -; CHECK: ret } define i16 @ctlz_i16(i16 %x) { -entry: +; CHECK-LABEL: ctlz_i16: +; CHECK: # BB#0: +; CHECK-NEXT: bsrw %di, %ax +; CHECK-NEXT: xorl $15, %eax +; CHECK-NEXT: retq %tmp2 = call i16 @llvm.ctlz.i16( i16 %x, i1 true ) ret i16 %tmp2 -; CHECK-LABEL: ctlz_i16: -; CHECK: bsrw -; CHECK-NOT: cmov -; CHECK: xorl $15, -; CHECK: ret } define i32 @ctlz_i32(i32 %x) { +; CHECK-LABEL: ctlz_i32: +; CHECK: # BB#0: +; CHECK-NEXT: bsrl %edi, %eax +; CHECK-NEXT: xorl $31, %eax +; CHECK-NEXT: retq %tmp = call i32 @llvm.ctlz.i32( i32 %x, i1 true ) ret i32 %tmp -; CHECK-LABEL: ctlz_i32: -; CHECK: bsrl -; CHECK-NOT: cmov -; CHECK: xorl $31, -; CHECK: ret } define i64 @ctlz_i64(i64 %x) { +; CHECK-LABEL: ctlz_i64: +; CHECK: # BB#0: +; CHECK-NEXT: bsrq %rdi, %rax +; CHECK-NEXT: xorq $63, %rax +; CHECK-NEXT: retq %tmp = call i64 @llvm.ctlz.i64( i64 %x, i1 true ) ret i64 %tmp -; CHECK-LABEL: ctlz_i64: -; CHECK: bsrq -; CHECK-NOT: cmov -; CHECK: xorq $63, -; CHECK: ret } -define i32 @ctlz_i32_cmov(i32 %n) { -entry: -; Generate a cmov to handle zero inputs when necessary. -; CHECK-LABEL: ctlz_i32_cmov: -; CHECK: bsrl -; CHECK: cmov -; CHECK: xorl $31, -; CHECK: ret +define i32 @ctlz_i32_zero_test(i32 %n) { +; Generate a test and branch to handle zero inputs because bsr/bsf are very slow. + +; CHECK-LABEL: ctlz_i32_zero_test: +; CHECK: # BB#0: +; CHECK-NEXT: movl $32, %eax +; CHECK-NEXT: testl %edi, %edi +; CHECK-NEXT: je .LBB8_2 +; CHECK-NEXT: # BB#1: # %cond.false +; CHECK-NEXT: bsrl %edi, %eax +; CHECK-NEXT: xorl $31, %eax +; CHECK-NEXT: .LBB8_2: # %cond.end +; CHECK-NEXT: retq %tmp1 = call i32 @llvm.ctlz.i32(i32 %n, i1 false) ret i32 %tmp1 } define i32 @ctlz_i32_fold_cmov(i32 %n) { -entry: ; Don't generate the cmovne when the source is known non-zero (and bsr would ; not set ZF). ; rdar://9490949 +; FIXME: The compare and branch are produced late in IR (by CodeGenPrepare), and +; codegen doesn't know how to delete the movl and je. + ; CHECK-LABEL: ctlz_i32_fold_cmov: -; CHECK: bsrl -; CHECK-NOT: cmov -; CHECK: xorl $31, -; CHECK: ret +; CHECK: # BB#0: +; CHECK-NEXT: orl $1, %edi +; CHECK-NEXT: movl $32, %eax +; CHECK-NEXT: je .LBB9_2 +; CHECK-NEXT: # BB#1: # %cond.false +; CHECK-NEXT: bsrl %edi, %eax +; CHECK-NEXT: xorl $31, %eax +; CHECK-NEXT: .LBB9_2: # %cond.end +; CHECK-NEXT: retq %or = or i32 %n, 1 %tmp1 = call i32 @llvm.ctlz.i32(i32 %or, i1 false) ret i32 %tmp1 } define i32 @ctlz_bsr(i32 %n) { -entry: ; Don't generate any xors when a 'ctlz' intrinsic is actually used to compute ; the most significant bit, which is what 'bsr' does natively. + ; CHECK-LABEL: ctlz_bsr: -; CHECK: bsrl -; CHECK-NOT: xorl -; CHECK: ret +; CHECK: # BB#0: +; CHECK-NEXT: bsrl %edi, %eax +; CHECK-NEXT: retq %ctlz = call i32 @llvm.ctlz.i32(i32 %n, i1 true) %bsr = xor i32 %ctlz, 31 ret i32 %bsr } -define i32 @ctlz_bsr_cmov(i32 %n) { -entry: -; Same as ctlz_bsr, but ensure this happens even when there is a potential -; zero. -; CHECK-LABEL: ctlz_bsr_cmov: -; CHECK: bsrl -; CHECK-NOT: xorl -; CHECK: ret +define i32 @ctlz_bsr_zero_test(i32 %n) { +; Generate a test and branch to handle zero inputs because bsr/bsf are very slow. +; FIXME: The compare and branch are produced late in IR (by CodeGenPrepare), and +; codegen doesn't know how to combine the $32 and $31 into $63. + +; CHECK-LABEL: ctlz_bsr_zero_test: +; CHECK: # BB#0: +; CHECK-NEXT: movl $32, %eax +; CHECK-NEXT: testl %edi, %edi +; CHECK-NEXT: je .LBB11_2 +; CHECK-NEXT: # BB#1: # %cond.false +; CHECK-NEXT: bsrl %edi, %eax +; CHECK-NEXT: xorl $31, %eax +; CHECK-NEXT: .LBB11_2: # %cond.end +; CHECK-NEXT: xorl $31, %eax +; CHECK-NEXT: retq %ctlz = call i32 @llvm.ctlz.i32(i32 %n, i1 false) %bsr = xor i32 %ctlz, 31 ret i32 %bsr diff --git a/test/CodeGen/X86/cmp.ll b/test/CodeGen/X86/cmp.ll index 584179aacbc9..eb9a29011428 100644 --- a/test/CodeGen/X86/cmp.ll +++ b/test/CodeGen/X86/cmp.ll @@ -211,3 +211,47 @@ define zeroext i1 @test15(i32 %bf.load, i32 %n) { ; CHECK: shrl $16, %edi ; CHECK: cmpl %esi, %edi } + +define i8 @test16(i16 signext %L) { + %lshr = lshr i16 %L, 15 + %trunc = trunc i16 %lshr to i8 + %not = xor i8 %trunc, 1 + ret i8 %not + +; CHECK-LABEL: test16: +; CHECK: testw %di, %di +; CHECK: setns %al +} + +define i8 @test17(i32 %L) { + %lshr = lshr i32 %L, 31 + %trunc = trunc i32 %lshr to i8 + %not = xor i8 %trunc, 1 + ret i8 %not + +; CHECK-LABEL: test17: +; CHECK: testl %edi, %edi +; CHECK: setns %al +} + +define i8 @test18(i64 %L) { + %lshr = lshr i64 %L, 63 + %trunc = trunc i64 %lshr to i8 + %not = xor i8 %trunc, 1 + ret i8 %not + +; CHECK-LABEL: test18: +; CHECK: testq %rdi, %rdi +; CHECK: setns %al +} + +define zeroext i1 @test19(i32 %L) { + %lshr = lshr i32 %L, 31 + %trunc = trunc i32 %lshr to i1 + %not = xor i1 %trunc, 1 + ret i1 %not + +; CHECK-LABEL: test19: +; CHECK: testl %edi, %edi +; CHECK: setns %al +} diff --git a/test/CodeGen/X86/cmpxchg-clobber-flags.ll b/test/CodeGen/X86/cmpxchg-clobber-flags.ll index 61123930887b..e21ba2a14cf5 100644 --- a/test/CodeGen/X86/cmpxchg-clobber-flags.ll +++ b/test/CodeGen/X86/cmpxchg-clobber-flags.ll @@ -1,24 +1,72 @@ -; RUN: llc -verify-machineinstrs -mtriple=i386-linux-gnu %s -o - | FileCheck %s -; RUN: llc -verify-machineinstrs -mtriple=i386-linux-gnu -pre-RA-sched=fast %s -o - | FileCheck %s -; RUN: llc -verify-machineinstrs -mtriple=x86_64-linux-gnu %s -o - | FileCheck %s -; RUN: llc -verify-machineinstrs -mtriple=x86_64-linux-gnu -pre-RA-sched=fast %s -o - | FileCheck %s +; RUN: llc -verify-machineinstrs -mtriple=i386-linux-gnu %s -o - | FileCheck %s -check-prefix=i386 +; RUN: llc -verify-machineinstrs -mtriple=i386-linux-gnu -pre-RA-sched=fast %s -o - | FileCheck %s -check-prefix=i386f -declare i32 @bar() +; RUN: llc -verify-machineinstrs -mtriple=x86_64-linux-gnu %s -o - | FileCheck %s -check-prefix=x8664 +; RUN: llc -verify-machineinstrs -mtriple=x86_64-linux-gnu -pre-RA-sched=fast %s -o - | FileCheck %s -check-prefix=x8664 +; RUN: llc -verify-machineinstrs -mtriple=x86_64-linux-gnu -mattr=+sahf %s -o - | FileCheck %s -check-prefix=x8664-sahf +; RUN: llc -verify-machineinstrs -mtriple=x86_64-linux-gnu -mattr=+sahf -pre-RA-sched=fast %s -o - | FileCheck %s -check-prefix=x8664-sahf +; RUN: llc -verify-machineinstrs -mtriple=x86_64-linux-gnu -mcpu=corei7 %s -o - | FileCheck %s -check-prefix=x8664-sahf -define i64 @test_intervening_call(i64* %foo, i64 %bar, i64 %baz) { -; CHECK-LABEL: test_intervening_call: -; CHECK: cmpxchg -; CHECK: pushf[[LQ:[lq]]] -; CHECK-NEXT: pop[[LQ]] [[FLAGS:%.*]] +declare i32 @foo() +declare i32 @bar(i64) -; CHECK-NEXT: call[[LQ]] bar +define i64 @test_intervening_call(i64* %foo, i64 %bar, i64 %baz) { +; i386-LABEL: test_intervening_call: +; i386: cmpxchg8b +; i386-NEXT: pushl %eax +; i386-NEXT: seto %al +; i386-NEXT: lahf +; i386-NEXT: movl %eax, [[FLAGS:%.*]] +; i386-NEXT: popl %eax +; i386-NEXT: movl %edx, 4(%esp) +; i386-NEXT: movl %eax, (%esp) +; i386-NEXT: calll bar +; i386-NEXT: movl [[FLAGS]], %eax +; i386-NEXT: addb $127, %al +; i386-NEXT: sahf +; i386-NEXT: jne + +; i386f-LABEL: test_intervening_call: +; i386f: cmpxchg8b +; i386f-NEXT: movl %eax, (%esp) +; i386f-NEXT: movl %edx, 4(%esp) +; i386f-NEXT: seto %al +; i386f-NEXT: lahf +; i386f-NEXT: movl %eax, [[FLAGS:%.*]] +; i386f-NEXT: calll bar +; i386f-NEXT: movl [[FLAGS]], %eax +; i386f-NEXT: addb $127, %al +; i386f-NEXT: sahf +; i386f-NEXT: jne + +; x8664-LABEL: test_intervening_call: +; x8664: cmpxchgq +; x8664: pushfq +; x8664-NEXT: popq [[FLAGS:%.*]] +; x8664-NEXT: movq %rax, %rdi +; x8664-NEXT: callq bar +; x8664-NEXT: pushq [[FLAGS]] +; x8664-NEXT: popfq +; x8664-NEXT: jne + +; x8664-sahf-LABEL: test_intervening_call: +; x8664-sahf: cmpxchgq +; x8664-sahf: pushq %rax +; x8664-sahf-NEXT: seto %al +; x8664-sahf-NEXT: lahf +; x8664-sahf-NEXT: movq %rax, [[FLAGS:%.*]] +; x8664-sahf-NEXT: popq %rax +; x8664-sahf-NEXT: movq %rax, %rdi +; x8664-sahf-NEXT: callq bar +; x8664-sahf-NEXT: movq [[FLAGS]], %rax +; x8664-sahf-NEXT: addb $127, %al +; x8664-sahf-NEXT: sahf +; x8664-sahf-NEXT: jne -; CHECK-NEXT: push[[LQ]] [[FLAGS]] -; CHECK-NEXT: popf[[LQ]] -; CHECK-NEXT: jne %cx = cmpxchg i64* %foo, i64 %bar, i64 %baz seq_cst seq_cst + %v = extractvalue { i64, i1 } %cx, 0 %p = extractvalue { i64, i1 } %cx, 1 - call i32 @bar() + call i32 @bar(i64 %v) br i1 %p, label %t, label %f t: @@ -30,10 +78,22 @@ f: ; Interesting in producing a clobber without any function calls. define i32 @test_control_flow(i32* %p, i32 %i, i32 %j) { -; CHECK-LABEL: test_control_flow: +; i386-LABEL: test_control_flow: +; i386: cmpxchg +; i386-NEXT: jne + +; i386f-LABEL: test_control_flow: +; i386f: cmpxchg +; i386f-NEXT: jne + +; x8664-LABEL: test_control_flow: +; x8664: cmpxchg +; x8664-NEXT: jne + +; x8664-sahf-LABEL: test_control_flow: +; x8664-sahf: cmpxchg +; x8664-sahf-NEXT: jne -; CHECK: cmpxchg -; CHECK-NEXT: jne entry: %cmp = icmp sgt i32 %i, %j br i1 %cmp, label %loop_start, label %cond.end @@ -67,20 +127,54 @@ cond.end: ; This one is an interesting case because CMOV doesn't have a chain ; operand. Naive attempts to limit cmpxchg EFLAGS use are likely to fail here. define i32 @test_feed_cmov(i32* %addr, i32 %desired, i32 %new) { -; CHECK-LABEL: test_feed_cmov: - -; CHECK: cmpxchg -; CHECK: pushf[[LQ:[lq]]] -; CHECK-NEXT: pop[[LQ]] [[FLAGS:%.*]] - -; CHECK-NEXT: call[[LQ]] bar +; i386-LABEL: test_feed_cmov: +; i386: cmpxchgl +; i386-NEXT: seto %al +; i386-NEXT: lahf +; i386-NEXT: movl %eax, [[FLAGS:%.*]] +; i386-NEXT: calll foo +; i386-NEXT: pushl %eax +; i386-NEXT: movl [[FLAGS]], %eax +; i386-NEXT: addb $127, %al +; i386-NEXT: sahf +; i386-NEXT: popl %eax + +; i386f-LABEL: test_feed_cmov: +; i386f: cmpxchgl +; i386f-NEXT: seto %al +; i386f-NEXT: lahf +; i386f-NEXT: movl %eax, [[FLAGS:%.*]] +; i386f-NEXT: calll foo +; i386f-NEXT: pushl %eax +; i386f-NEXT: movl [[FLAGS]], %eax +; i386f-NEXT: addb $127, %al +; i386f-NEXT: sahf +; i386f-NEXT: popl %eax + +; x8664-LABEL: test_feed_cmov: +; x8664: cmpxchg +; x8664: pushfq +; x8664-NEXT: popq [[FLAGS:%.*]] +; x8664-NEXT: callq foo +; x8664-NEXT: pushq [[FLAGS]] +; x8664-NEXT: popfq + +; x8664-sahf-LABEL: test_feed_cmov: +; x8664-sahf: cmpxchgl +; x8664-sahf: seto %al +; x8664-sahf-NEXT: lahf +; x8664-sahf-NEXT: movq %rax, [[FLAGS:%.*]] +; x8664-sahf-NEXT: callq foo +; x8664-sahf-NEXT: pushq %rax +; x8664-sahf-NEXT: movq [[FLAGS]], %rax +; x8664-sahf-NEXT: addb $127, %al +; x8664-sahf-NEXT: sahf +; x8664-sahf-NEXT: popq %rax -; CHECK-NEXT: push[[LQ]] [[FLAGS]] -; CHECK-NEXT: popf[[LQ]] %res = cmpxchg i32* %addr, i32 %desired, i32 %new seq_cst seq_cst %success = extractvalue { i32, i1 } %res, 1 - %rhs = call i32 @bar() + %rhs = call i32 @foo() %ret = select i1 %success, i32 %new, i32 %rhs ret i32 %ret diff --git a/test/CodeGen/X86/coal-sections.ll b/test/CodeGen/X86/coal-sections.ll new file mode 100644 index 000000000000..05b2a8c8bf87 --- /dev/null +++ b/test/CodeGen/X86/coal-sections.ll @@ -0,0 +1,23 @@ +; RUN: llc < %s -mtriple x86_64-apple-darwin | FileCheck %s + +; Check that *coal* sections are not emitted. + +; CHECK: .section __TEXT,__text,regular,pure_instructions{{$}} +; CHECK-NEXT: .globl _foo + +; CHECK: .section __TEXT,__const{{$}} +; CHECK-NEXT: .globl _a + +; CHECK: .section __DATA,__data{{$}} +; CHECK-NEXT: .globl _b + +@a = weak_odr constant [4 x i32] [i32 1, i32 2, i32 3, i32 4], align 16 +@b = weak global i32 5, align 4 +@g = common global i32* null, align 8 + +; Function Attrs: nounwind ssp uwtable +define weak i32* @foo() { +entry: + store i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i64 0), i32** @g, align 8 + ret i32* @b +} diff --git a/test/CodeGen/X86/coalescer-win64.ll b/test/CodeGen/X86/coalescer-win64.ll new file mode 100644 index 000000000000..ff084ae5b9e0 --- /dev/null +++ b/test/CodeGen/X86/coalescer-win64.ll @@ -0,0 +1,16 @@ +; RUN: llc < %s -verify-coalescing | FileCheck %s +target triple = "x86_64-pc-win32" + +@fnptr = external global void ()* + +define void @test1() { +entry: + %p = load void ()*, void ()** @fnptr + tail call void %p() + ret void +} + +; CHECK-LABEL: test1{{$}} +; CHECK: .seh_proc test1{{$}} +; CHECK: rex64 jmpq *fnptr(%rip) +; CHECK: .seh_endproc diff --git a/test/CodeGen/X86/code_placement_cold_loop_blocks.ll b/test/CodeGen/X86/code_placement_cold_loop_blocks.ll new file mode 100644 index 000000000000..592d1ce45bb6 --- /dev/null +++ b/test/CodeGen/X86/code_placement_cold_loop_blocks.ll @@ -0,0 +1,122 @@ +; RUN: llc -mcpu=corei7 -mtriple=x86_64-linux < %s | FileCheck %s -check-prefix=CHECK + +define void @foo() !prof !1 { +; Test if a cold block in a loop will be placed at the end of the function +; chain. +; +; CHECK-LABEL: foo: +; CHECK: callq b +; CHECK: callq c +; CHECK: callq e +; CHECK: callq f +; CHECK: callq d + +entry: + br label %header + +header: + call void @b() + %call = call zeroext i1 @a() + br i1 %call, label %if.then, label %if.else, !prof !4 + +if.then: + call void @c() + br label %if.end + +if.else: + call void @d() + br label %if.end + +if.end: + call void @e() + %call2 = call zeroext i1 @a() + br i1 %call2, label %header, label %end, !prof !5 + +end: + call void @f() + ret void +} + +define void @nested_loop_0() !prof !1 { +; Test if a block that is cold in the inner loop but not cold in the outer loop +; will merged to the outer loop chain. +; +; CHECK-LABEL: nested_loop_0: +; CHECK: callq c +; CHECK: callq d +; CHECK: callq e +; CHECK: callq b +; CHECK: callq f + +entry: + br label %header + +header: + call void @b() + %call4 = call zeroext i1 @a() + br i1 %call4, label %header2, label %end + +header2: + call void @c() + %call = call zeroext i1 @a() + br i1 %call, label %if.then, label %if.else, !prof !2 + +if.then: + call void @d() + %call3 = call zeroext i1 @a() + br i1 %call3, label %header2, label %header, !prof !3 + +if.else: + call void @e() + %call2 = call zeroext i1 @a() + br i1 %call2, label %header2, label %header, !prof !3 + +end: + call void @f() + ret void +} + +define void @nested_loop_1() !prof !1 { +; Test if a cold block in an inner loop will be placed at the end of the +; function chain. +; +; CHECK-LABEL: nested_loop_1: +; CHECK: callq b +; CHECK: callq c +; CHECK: callq e +; CHECK: callq d + +entry: + br label %header + +header: + call void @b() + br label %header2 + +header2: + call void @c() + %call = call zeroext i1 @a() + br i1 %call, label %end, label %if.else, !prof !4 + +if.else: + call void @d() + %call2 = call zeroext i1 @a() + br i1 %call2, label %header2, label %header, !prof !5 + +end: + call void @e() + ret void +} + +declare zeroext i1 @a() +declare void @b() +declare void @c() +declare void @d() +declare void @e() +declare void @f() + +!1 = !{!"function_entry_count", i64 1} +!2 = !{!"branch_weights", i32 100, i32 1} +!3 = !{!"branch_weights", i32 1, i32 10} +!4 = !{!"branch_weights", i32 1000, i32 1} +!5 = !{!"branch_weights", i32 100, i32 1} diff --git a/test/CodeGen/X86/code_placement_ignore_succ_in_inner_loop.ll b/test/CodeGen/X86/code_placement_ignore_succ_in_inner_loop.ll new file mode 100644 index 000000000000..79b4883fb1d6 --- /dev/null +++ b/test/CodeGen/X86/code_placement_ignore_succ_in_inner_loop.ll @@ -0,0 +1,123 @@ +; RUN: llc -mcpu=corei7 -mtriple=x86_64-linux < %s | FileCheck %s -check-prefix=CHECK + +define void @foo() { +; Test that when determining the edge probability from a node in an inner loop +; to a node in an outer loop, the weights on edges in the inner loop should be +; ignored if we are building the chain for the outer loop. +; +; CHECK-LABEL: foo: +; CHECK: callq c +; CHECK: callq b + +entry: + %call = call zeroext i1 @a() + br i1 %call, label %if.then, label %if.else, !prof !1 + +if.then: + %call1 = call zeroext i1 @a() + br i1 %call1, label %while.body, label %if.end.1, !prof !1 + +while.body: + %call2 = call zeroext i1 @a() + br i1 %call2, label %if.then.1, label %while.cond + +if.then.1: + call void @d() + br label %while.cond + +while.cond: + %call3 = call zeroext i1 @a() + br i1 %call3, label %while.body, label %if.end + +if.end.1: + call void @d() + br label %if.end + +if.else: + call void @b() + br label %if.end + +if.end: + call void @c() + ret void +} + +define void @bar() { +; Test that when determining the edge probability from a node in a loop to a +; node in its peer loop, the weights on edges in the first loop should be +; ignored. +; +; CHECK-LABEL: bar: +; CHECK: callq c +; CHECK: callq b + +entry: + %call = call zeroext i1 @a() + br i1 %call, label %if.then, label %if.else, !prof !1 + +if.then: + %call1 = call zeroext i1 @a() + br i1 %call1, label %if.then, label %while.body, !prof !2 + +while.body: + %call2 = call zeroext i1 @a() + br i1 %call2, label %while.body, label %if.end, !prof !2 + +if.else: + call void @b() + br label %if.end + +if.end: + call void @c() + ret void +} + +define void @par() { +; Test that when determining the edge probability from a node in a loop to a +; node in its outer loop, the weights on edges in the outer loop should be +; ignored if we are building the chain for the inner loop. +; +; CHECK-LABEL: par: +; CHECK: callq c +; CHECK: callq d +; CHECK: callq b + +entry: + br label %if.cond + +if.cond: + %call = call zeroext i1 @a() + br i1 %call, label %if.then, label %if.else, !prof !3 + +if.then: + call void @b() + br label %if.end + +if.else: + call void @c() + %call1 = call zeroext i1 @a() + br i1 %call1, label %if.end, label %exit, !prof !4 + +if.end: + call void @d() + %call2 = call zeroext i1 @a() + br i1 %call2, label %if.cond, label %if.end.2, !prof !2 + +if.end.2: + call void @e() + br label %if.cond + +exit: + ret void +} + +declare zeroext i1 @a() +declare void @b() +declare void @c() +declare void @d() +declare void @e() + +!1 = !{!"branch_weights", i32 10, i32 1} +!2 = !{!"branch_weights", i32 100, i32 1} +!3 = !{!"branch_weights", i32 1, i32 100} +!4 = !{!"branch_weights", i32 1, i32 1} diff --git a/test/CodeGen/X86/code_placement_loop_rotation.ll b/test/CodeGen/X86/code_placement_loop_rotation.ll new file mode 100644 index 000000000000..3ec5961486e8 --- /dev/null +++ b/test/CodeGen/X86/code_placement_loop_rotation.ll @@ -0,0 +1,80 @@ +; RUN: llc -mcpu=corei7 -mtriple=x86_64-linux < %s | FileCheck %s -check-prefix=CHECK +; RUN: llc -mcpu=corei7 -mtriple=x86_64-linux -precise-rotation-cost < %s | FileCheck %s -check-prefix=CHECK-PROFILE + +define void @foo() { +; Test that not all edges in the loop chain are fall through without profile +; data. +; +; CHECK-LABEL: foo: +; CHECK: callq e +; CHECK: callq f +; CHECK: callq g +; CHECK: callq h + +entry: + br label %header + +header: + call void @e() + %call = call zeroext i1 @a() + br i1 %call, label %if.then, label %if.else, !prof !2 + +if.then: + call void @f() + br label %if.end + +if.else: + call void @g() + br label %if.end + +if.end: + call void @h() + %call2 = call zeroext i1 @a() + br i1 %call2, label %header, label %end + +end: + ret void +} + +define void @bar() !prof !1 { +; Test that all edges in the loop chain are fall through with profile data. +; +; CHECK-PROFILE-LABEL: bar: +; CHECK-PROFILE: callq g +; CHECK-PROFILE: callq h +; CHECK-PROFILE: callq e +; CHECK-PROFILE: callq f + +entry: + br label %header + +header: + call void @e() + %call = call zeroext i1 @a() + br i1 %call, label %if.then, label %if.else, !prof !2 + +if.then: + call void @f() + br label %if.end + +if.else: + call void @g() + br label %if.end + +if.end: + call void @h() + %call2 = call zeroext i1 @a() + br i1 %call2, label %header, label %end + +end: + ret void +} + +declare zeroext i1 @a() +declare void @e() +declare void @f() +declare void @g() +declare void @h() + +!1 = !{!"function_entry_count", i64 1} +!2 = !{!"branch_weights", i32 16, i32 16} diff --git a/test/CodeGen/X86/code_placement_loop_rotation2.ll b/test/CodeGen/X86/code_placement_loop_rotation2.ll new file mode 100644 index 000000000000..6d8b3c99cd05 --- /dev/null +++ b/test/CodeGen/X86/code_placement_loop_rotation2.ll @@ -0,0 +1,122 @@ +; RUN: llc -mcpu=corei7 -mtriple=x86_64-linux < %s | FileCheck %s -check-prefix=CHECK +; RUN: llc -mcpu=corei7 -mtriple=x86_64-linux -precise-rotation-cost < %s | FileCheck %s -check-prefix=CHECK-PROFILE + +define void @foo() { +; Test a nested loop case when profile data is not available. +; +; CHECK-LABEL: foo: +; CHECK: callq b +; CHECK: callq c +; CHECK: callq d +; CHECK: callq e +; CHECK: callq f +; CHECK: callq g +; CHECK: callq h + +entry: + br label %header + +header: + call void @b() + %call = call zeroext i1 @a() + br i1 %call, label %if.then, label %if.else, !prof !2 + +if.then: + br label %header2 + +header2: + call void @c() + %call1 = call zeroext i1 @a() + br i1 %call1, label %if.then2, label %if.else2, !prof !2 + +if.then2: + call void @d() + br label %if.end2 + +if.else2: + call void @e() + br label %if.end2 + +if.end2: + call void @f() + %call2 = call zeroext i1 @a() + br i1 %call2, label %header2, label %if.end + +if.else: + call void @g() + br label %if.end + +if.end: + call void @h() + %call3 = call zeroext i1 @a() + br i1 %call3, label %header, label %end + +end: + ret void +} + +define void @bar() !prof !1 { +; Test a nested loop case when profile data is available. +; +; CHECK-PROFILE-LABEL: bar: +; CHECK-PROFILE: callq e +; CHECK-PROFILE: callq f +; CHECK-PROFILE: callq c +; CHECK-PROFILE: callq d +; CHECK-PROFILE: callq h +; CHECK-PROFILE: callq b +; CHECK-PROFILE: callq g + +entry: + br label %header + +header: + call void @b() + %call = call zeroext i1 @a() + br i1 %call, label %if.then, label %if.else, !prof !2 + +if.then: + br label %header2 + +header2: + call void @c() + %call1 = call zeroext i1 @a() + br i1 %call1, label %if.then2, label %if.else2, !prof !2 + +if.then2: + call void @d() + br label %if.end2 + +if.else2: + call void @e() + br label %if.end2 + +if.end2: + call void @f() + %call2 = call zeroext i1 @a() + br i1 %call2, label %header2, label %if.end + +if.else: + call void @g() + br label %if.end + +if.end: + call void @h() + %call3 = call zeroext i1 @a() + br i1 %call3, label %header, label %end + +end: + ret void +} + +declare zeroext i1 @a() +declare void @b() +declare void @c() +declare void @d() +declare void @e() +declare void @f() +declare void @g() +declare void @h() + +!1 = !{!"function_entry_count", i64 1} +!2 = !{!"branch_weights", i32 16, i32 16} diff --git a/test/CodeGen/X86/codegen-prepare-cast.ll b/test/CodeGen/X86/codegen-prepare-cast.ll index 1ab8017e8858..c5c2d64f63d8 100644 --- a/test/CodeGen/X86/codegen-prepare-cast.ll +++ b/test/CodeGen/X86/codegen-prepare-cast.ll @@ -11,7 +11,7 @@ target triple = "x86_64-unknown-linux-gnu" ; CHECK-LABEL: @_Dmain ; CHECK: load i8, i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str, i32 0, i32 0) -; CHECK ret +; CHECK: ret define fastcc i32 @_Dmain(%"char[][]" %unnamed) { entry: %tmp = getelementptr [7 x i8], [7 x i8]* @.str, i32 0, i32 0 ; [#uses=1] diff --git a/test/CodeGen/X86/coff-comdat.ll b/test/CodeGen/X86/coff-comdat.ll index 18f418959ec9..712825a99100 100644 --- a/test/CodeGen/X86/coff-comdat.ll +++ b/test/CodeGen/X86/coff-comdat.ll @@ -53,7 +53,7 @@ define x86_fastcallcc void @f8() comdat($f8) { $vftable = comdat largest @some_name = private unnamed_addr constant [2 x i8*] zeroinitializer, comdat($vftable) -@vftable = alias getelementptr([2 x i8*], [2 x i8*]* @some_name, i32 0, i32 1) +@vftable = alias i8*, getelementptr([2 x i8*], [2 x i8*]* @some_name, i32 0, i32 1) ; CHECK: .section .text,"xr",discard,_f1 ; CHECK: .globl _f1 diff --git a/test/CodeGen/X86/combine-and.ll b/test/CodeGen/X86/combine-and.ll index bb46ac539171..fddf18d1bdb0 100644 --- a/test/CodeGen/X86/combine-and.ll +++ b/test/CodeGen/X86/combine-and.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 < %s | FileCheck %s ; ; Verify that the DAGCombiner is able to fold a vector AND into a blend diff --git a/test/CodeGen/X86/combine-avx-intrinsics.ll b/test/CodeGen/X86/combine-avx-intrinsics.ll index f610f7fcb91e..64e081523c1f 100644 --- a/test/CodeGen/X86/combine-avx-intrinsics.ll +++ b/test/CodeGen/X86/combine-avx-intrinsics.ll @@ -19,24 +19,6 @@ define <8 x float> @test_x86_avx_blend_ps_256(<8 x float> %a0) { ; CHECK: ret -define <4 x double> @test_x86_avx_blendv_pd_256(<4 x double> %a0, <4 x double> %a1) { - %1 = call <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double> %a0, <4 x double> %a0, <4 x double> %a1) - ret <4 x double> %1 -} -; CHECK-LABEL: test_x86_avx_blendv_pd_256 -; CHECK-NOT: vblendvpd -; CHECK: ret - - -define <8 x float> @test_x86_avx_blendv_ps_256(<8 x float> %a0, <8 x float> %a1) { - %1 = call <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float> %a0, <8 x float> %a0, <8 x float> %a1) - ret <8 x float> %1 -} -; CHECK-LABEL: test_x86_avx_blendv_ps_256 -; CHECK-NOT: vblendvps -; CHECK: ret - - define <4 x double> @test2_x86_avx_blend_pd_256(<4 x double> %a0, <4 x double> %a1) { %1 = call <4 x double> @llvm.x86.avx.blend.pd.256(<4 x double> %a0, <4 x double> %a1, i32 0) ret <4 x double> %1 @@ -55,24 +37,6 @@ define <8 x float> @test2_x86_avx_blend_ps_256(<8 x float> %a0, <8 x float> %a1) ; CHECK: ret -define <4 x double> @test2_x86_avx_blendv_pd_256(<4 x double> %a0, <4 x double> %a1) { - %1 = call <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> zeroinitializer) - ret <4 x double> %1 -} -; CHECK-LABEL: test2_x86_avx_blendv_pd_256 -; CHECK-NOT: vblendvpd -; CHECK: ret - - -define <8 x float> @test2_x86_avx_blendv_ps_256(<8 x float> %a0, <8 x float> %a1) { - %1 = call <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> zeroinitializer) - ret <8 x float> %1 -} -; CHECK-LABEL: test2_x86_avx_blendv_ps_256 -; CHECK-NOT: vblendvps -; CHECK: ret - - define <4 x double> @test3_x86_avx_blend_pd_256(<4 x double> %a0, <4 x double> %a1) { %1 = call <4 x double> @llvm.x86.avx.blend.pd.256(<4 x double> %a0, <4 x double> %a1, i32 -1) ret <4 x double> %1 @@ -91,29 +55,6 @@ define <8 x float> @test3_x86_avx_blend_ps_256(<8 x float> %a0, <8 x float> %a1) ; CHECK: ret -define <4 x double> @test3_x86_avx_blendv_pd_256(<4 x double> %a0, <4 x double> %a1) { - %Mask = bitcast <4 x i64> to <4 x double> - %1 = call <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %Mask) - ret <4 x double> %1 -} -; CHECK-LABEL: test3_x86_avx_blendv_pd_256 -; CHECK-NOT: vblendvpd -; CHECK: ret - - -define <8 x float> @test3_x86_avx_blendv_ps_256(<8 x float> %a0, <8 x float> %a1) { - %Mask = bitcast <4 x i64> to <8 x float> - %1 = call <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %Mask) - ret <8 x float> %1 -} -; CHECK-LABEL: test3_x86_avx_blendv_ps_256 -; CHECK-NOT: vblendvps -; CHECK: ret - - - declare <4 x double> @llvm.x86.avx.blend.pd.256(<4 x double>, <4 x double>, i32) declare <8 x float> @llvm.x86.avx.blend.ps.256(<8 x float>, <8 x float>, i32) -declare <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double>, <4 x double>, <4 x double>) -declare <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float>, <8 x float>, <8 x float>) diff --git a/test/CodeGen/X86/combine-avx2-intrinsics.ll b/test/CodeGen/X86/combine-avx2-intrinsics.ll index 8794f8b86849..2714b26c9141 100644 --- a/test/CodeGen/X86/combine-avx2-intrinsics.ll +++ b/test/CodeGen/X86/combine-avx2-intrinsics.ll @@ -3,56 +3,6 @@ ; Verify that the backend correctly combines AVX2 builtin intrinsics. -define <8 x i32> @test_psra_1(<8 x i32> %A) { - %1 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %A, i32 3) - %2 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %1, <4 x i32> ) - %3 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %2, i32 2) - ret <8 x i32> %3 -} -; CHECK-LABEL: test_psra_1 -; CHECK: vpsrad $8, %ymm0, %ymm0 -; CHECK-NEXT: ret - -define <16 x i16> @test_psra_2(<16 x i16> %A) { - %1 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %A, i32 3) - %2 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %1, <8 x i16> ) - %3 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %2, i32 2) - ret <16 x i16> %3 -} -; CHECK-LABEL: test_psra_2 -; CHECK: vpsraw $8, %ymm0, %ymm0 -; CHECK-NEXT: ret - -define <16 x i16> @test_psra_3(<16 x i16> %A) { - %1 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %A, i32 0) - %2 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %1, <8 x i16> ) - %3 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %2, i32 0) - ret <16 x i16> %3 -} -; CHECK-LABEL: test_psra_3 -; CHECK-NOT: vpsraw -; CHECK: ret - -define <8 x i32> @test_psra_4(<8 x i32> %A) { - %1 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %A, i32 0) - %2 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %1, <4 x i32> ) - %3 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %2, i32 0) - ret <8 x i32> %3 -} -; CHECK-LABEL: test_psra_4 -; CHECK-NOT: vpsrad -; CHECK: ret - - -define <32 x i8> @test_x86_avx2_pblendvb(<32 x i8> %a0, <32 x i8> %a1) { - %res = call <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8> %a0, <32 x i8> %a0, <32 x i8> %a1) - ret <32 x i8> %res -} -; CHECK-LABEL: test_x86_avx2_pblendvb -; CHECK-NOT: vpblendvb -; CHECK: ret - - define <16 x i16> @test_x86_avx2_pblendw(<16 x i16> %a0) { %res = call <16 x i16> @llvm.x86.avx2.pblendw(<16 x i16> %a0, <16 x i16> %a0, i32 7) ret <16 x i16> %res @@ -80,15 +30,6 @@ define <8 x i32> @test_x86_avx2_pblendd_256(<8 x i32> %a0) { ; CHECK: ret -define <32 x i8> @test2_x86_avx2_pblendvb(<32 x i8> %a0, <32 x i8> %a1) { - %res = call <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> zeroinitializer) - ret <32 x i8> %res -} -; CHECK-LABEL: test2_x86_avx2_pblendvb -; CHECK-NOT: vpblendvb -; CHECK: ret - - define <16 x i16> @test2_x86_avx2_pblendw(<16 x i16> %a0, <16 x i16> %a1) { %res = call <16 x i16> @llvm.x86.avx2.pblendw(<16 x i16> %a0, <16 x i16> %a1, i32 0) ret <16 x i16> %res @@ -116,16 +57,6 @@ define <8 x i32> @test2_x86_avx2_pblendd_256(<8 x i32> %a0, <8 x i32> %a1) { ; CHECK: ret -define <32 x i8> @test3_x86_avx2_pblendvb(<32 x i8> %a0, <32 x i8> %a1) { - %1 = bitcast <4 x i64> to <32 x i8> - %res = call <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> %1) - ret <32 x i8> %res -} -; CHECK-LABEL: test3_x86_avx2_pblendvb -; CHECK-NOT: vpblendvb -; CHECK: ret - - define <16 x i16> @test3_x86_avx2_pblendw(<16 x i16> %a0, <16 x i16> %a1) { %res = call <16 x i16> @llvm.x86.avx2.pblendw(<16 x i16> %a0, <16 x i16> %a1, i32 -1) ret <16 x i16> %res @@ -153,12 +84,7 @@ define <8 x i32> @test3_x86_avx2_pblendd_256(<8 x i32> %a0, <8 x i32> %a1) { ; CHECK: ret -declare <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8>, <32 x i8>, <32 x i8>) declare <16 x i16> @llvm.x86.avx2.pblendw(<16 x i16>, <16 x i16>, i32) declare <4 x i32> @llvm.x86.avx2.pblendd.128(<4 x i32>, <4 x i32>, i32) declare <8 x i32> @llvm.x86.avx2.pblendd.256(<8 x i32>, <8 x i32>, i32) -declare <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16>, <8 x i16>) -declare <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16>, i32) -declare <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32>, <4 x i32>) -declare <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32>, i32) diff --git a/test/CodeGen/X86/combine-multiplies.ll b/test/CodeGen/X86/combine-multiplies.ll new file mode 100644 index 000000000000..5e51edbf52f9 --- /dev/null +++ b/test/CodeGen/X86/combine-multiplies.ll @@ -0,0 +1,163 @@ +; RUN: llc < %s -mattr=sse2 -mtriple=i386-unknown-linux-gnu | FileCheck %s + +; Source file looks something like this: +; +; typedef int AAA[100][100]; +; +; void testCombineMultiplies(AAA a,int lll) +; { +; int LOC = lll + 5; +; +; a[LOC][LOC] = 11; +; +; a[LOC][20] = 22; +; a[LOC+20][20] = 33; +; } +; +; We want to make sure we don't generate 2 multiply instructions, +; one for a[LOC][] and one for a[LOC+20]. visitMUL in DAGCombiner.cpp +; should combine the instructions in such a way to avoid the extra +; multiply. +; +; Output looks roughly like this: +; +; movl 8(%esp), %eax +; movl 12(%esp), %ecx +; imull $400, %ecx, %edx # imm = 0x190 +; leal (%edx,%eax), %esi +; movl $11, 2020(%esi,%ecx,4) +; movl $22, 2080(%edx,%eax) +; movl $33, 10080(%edx,%eax) +; +; CHECK-LABEL: testCombineMultiplies +; CHECK: imull $400, [[ARG1:%[a-z]+]], [[MUL:%[a-z]+]] # imm = 0x190 +; CHECK-NEXT: leal ([[MUL]],[[ARG2:%[a-z]+]]), [[LEA:%[a-z]+]] +; CHECK-NEXT: movl $11, {{[0-9]+}}([[LEA]],[[ARG1]],4) +; CHECK-NEXT: movl $22, {{[0-9]+}}([[MUL]],[[ARG2]]) +; CHECK-NEXT: movl $33, {{[0-9]+}}([[MUL]],[[ARG2]]) +; CHECK: retl +; + +; Function Attrs: nounwind +define void @testCombineMultiplies([100 x i32]* nocapture %a, i32 %lll) { +entry: + %add = add nsw i32 %lll, 5 + %arrayidx1 = getelementptr inbounds [100 x i32], [100 x i32]* %a, i32 %add, i32 %add + store i32 11, i32* %arrayidx1, align 4 + %arrayidx3 = getelementptr inbounds [100 x i32], [100 x i32]* %a, i32 %add, i32 20 + store i32 22, i32* %arrayidx3, align 4 + %add4 = add nsw i32 %lll, 25 + %arrayidx6 = getelementptr inbounds [100 x i32], [100 x i32]* %a, i32 %add4, i32 20 + store i32 33, i32* %arrayidx6, align 4 + ret void +} + + +; Test for the same optimization on vector multiplies. +; +; Source looks something like this: +; +; typedef int v4int __attribute__((__vector_size__(16))); +; +; v4int x; +; v4int v2, v3; +; void testCombineMultiplies_splat(v4int v1) { +; v2 = (v1 + (v4int){ 11, 11, 11, 11 }) * (v4int) {22, 22, 22, 22}; +; v3 = (v1 + (v4int){ 33, 33, 33, 33 }) * (v4int) {22, 22, 22, 22}; +; x = (v1 + (v4int){ 11, 11, 11, 11 }); +; } +; +; Output looks something like this: +; +; testCombineMultiplies_splat: # @testCombineMultiplies_splat +; # BB#0: # %entry +; movdqa .LCPI1_0, %xmm1 # xmm1 = [11,11,11,11] +; paddd %xmm0, %xmm1 +; movdqa .LCPI1_1, %xmm2 # xmm2 = [22,22,22,22] +; pshufd $245, %xmm0, %xmm3 # xmm3 = xmm0[1,1,3,3] +; pmuludq %xmm2, %xmm0 +; pshufd $232, %xmm0, %xmm0 # xmm0 = xmm0[0,2,2,3] +; pmuludq %xmm2, %xmm3 +; pshufd $232, %xmm3, %xmm2 # xmm2 = xmm3[0,2,2,3] +; punpckldq %xmm2, %xmm0 # xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; movdqa .LCPI1_2, %xmm2 # xmm2 = [242,242,242,242] +; paddd %xmm0, %xmm2 +; paddd .LCPI1_3, %xmm0 +; movdqa %xmm2, v2 +; movdqa %xmm0, v3 +; movdqa %xmm1, x +; retl +; +; Again, we want to make sure we don't generate two different multiplies. +; We should have a single multiply for "v1 * {22, 22, 22, 22}" (made up of two +; pmuludq instructions), followed by two adds. Without this optimization, we'd +; do 2 adds, followed by 2 multiplies (i.e. 4 pmuludq instructions). +; +; CHECK-LABEL: testCombineMultiplies_splat +; CHECK: movdqa .LCPI1_0, [[C11:%xmm[0-9]]] +; CHECK-NEXT: paddd %xmm0, [[C11]] +; CHECK-NEXT: movdqa .LCPI1_1, [[C22:%xmm[0-9]]] +; CHECK-NEXT: pshufd $245, %xmm0, [[T1:%xmm[0-9]]] +; CHECK-NEXT: pmuludq [[C22]], [[T2:%xmm[0-9]]] +; CHECK-NEXT: pshufd $232, [[T2]], [[T3:%xmm[0-9]]] +; CHECK-NEXT: pmuludq [[C22]], [[T4:%xmm[0-9]]] +; CHECK-NEXT: pshufd $232, [[T4]], [[T5:%xmm[0-9]]] +; CHECK-NEXT: punpckldq [[T5]], [[T6:%xmm[0-9]]] +; CHECK-NEXT: movdqa .LCPI1_2, [[C242:%xmm[0-9]]] +; CHECK-NEXT: paddd [[T6]], [[C242]] +; CHECK-NEXT: paddd .LCPI1_3, [[C726:%xmm[0-9]]] +; CHECK-NEXT: movdqa [[C242]], v2 +; CHECK-NEXT: [[C726]], v3 +; CHECK-NEXT: [[C11]], x +; CHECK-NEXT: retl + +@v2 = common global <4 x i32> zeroinitializer, align 16 +@v3 = common global <4 x i32> zeroinitializer, align 16 +@x = common global <4 x i32> zeroinitializer, align 16 + +; Function Attrs: nounwind +define void @testCombineMultiplies_splat(<4 x i32> %v1) { +entry: + %add1 = add <4 x i32> %v1, + %mul1 = mul <4 x i32> %add1, + %add2 = add <4 x i32> %v1, + %mul2 = mul <4 x i32> %add2, + store <4 x i32> %mul1, <4 x i32>* @v2, align 16 + store <4 x i32> %mul2, <4 x i32>* @v3, align 16 + store <4 x i32> %add1, <4 x i32>* @x, align 16 + ret void +} + +; Finally, check the non-splatted vector case. This is very similar +; to the previous test case, except for the vector values. +; +; CHECK-LABEL: testCombineMultiplies_non_splat +; CHECK: movdqa .LCPI2_0, [[C11:%xmm[0-9]]] +; CHECK-NEXT: paddd %xmm0, [[C11]] +; CHECK-NEXT: movdqa .LCPI2_1, [[C22:%xmm[0-9]]] +; CHECK-NEXT: pshufd $245, %xmm0, [[T1:%xmm[0-9]]] +; CHECK-NEXT: pmuludq [[C22]], [[T2:%xmm[0-9]]] +; CHECK-NEXT: pshufd $232, [[T2]], [[T3:%xmm[0-9]]] +; CHECK-NEXT: pshufd $245, [[C22]], [[T7:%xmm[0-9]]] +; CHECK-NEXT: pmuludq [[T1]], [[T7]] +; CHECK-NEXT: pshufd $232, [[T7]], [[T5:%xmm[0-9]]] +; CHECK-NEXT: punpckldq [[T5]], [[T6:%xmm[0-9]]] +; CHECK-NEXT: movdqa .LCPI2_2, [[C242:%xmm[0-9]]] +; CHECK-NEXT: paddd [[T6]], [[C242]] +; CHECK-NEXT: paddd .LCPI2_3, [[C726:%xmm[0-9]]] +; CHECK-NEXT: movdqa [[C242]], v2 +; CHECK-NEXT: [[C726]], v3 +; CHECK-NEXT: [[C11]], x +; CHECK-NEXT: retl +; Function Attrs: nounwind +define void @testCombineMultiplies_non_splat(<4 x i32> %v1) { +entry: + %add1 = add <4 x i32> %v1, + %mul1 = mul <4 x i32> %add1, + %add2 = add <4 x i32> %v1, + %mul2 = mul <4 x i32> %add2, + store <4 x i32> %mul1, <4 x i32>* @v2, align 16 + store <4 x i32> %mul2, <4 x i32>* @v3, align 16 + store <4 x i32> %add1, <4 x i32>* @x, align 16 + ret void +} diff --git a/test/CodeGen/X86/combine-or.ll b/test/CodeGen/X86/combine-or.ll index 970f1762c1b8..e17cfbeeee12 100644 --- a/test/CodeGen/X86/combine-or.ll +++ b/test/CodeGen/X86/combine-or.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 | FileCheck %s diff --git a/test/CodeGen/X86/combine-sse2-intrinsics.ll b/test/CodeGen/X86/combine-sse2-intrinsics.ll deleted file mode 100644 index fa500e5d8d67..000000000000 --- a/test/CodeGen/X86/combine-sse2-intrinsics.ll +++ /dev/null @@ -1,53 +0,0 @@ -; RUN: llc < %s -march=x86 -mcpu=core2 | FileCheck %s -; RUN: llc < %s -march=x86-64 -mcpu=corei7 | FileCheck %s - -; Verify that the backend correctly combines SSE2 builtin intrinsics. - - -define <4 x i32> @test_psra_1(<4 x i32> %A) { - %1 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %A, i32 3) - %2 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %1, <4 x i32> ) - %3 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %2, i32 2) - ret <4 x i32> %3 -} -; CHECK-LABEL: test_psra_1 -; CHECK: psrad $8, %xmm0 -; CHECK-NEXT: ret - -define <8 x i16> @test_psra_2(<8 x i16> %A) { - %1 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %A, i32 3) - %2 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %1, <8 x i16> ) - %3 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %2, i32 2) - ret <8 x i16> %3 -} -; CHECK-LABEL: test_psra_2 -; CHECK: psraw $8, %xmm0 -; CHECK-NEXT: ret - -define <4 x i32> @test_psra_3(<4 x i32> %A) { - %1 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %A, i32 0) - %2 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %1, <4 x i32> ) - %3 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %2, i32 0) - ret <4 x i32> %3 -} -; CHECK-LABEL: test_psra_3 -; CHECK-NOT: psrad -; CHECK: ret - - -define <8 x i16> @test_psra_4(<8 x i16> %A) { - %1 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %A, i32 0) - %2 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %1, <8 x i16> ) - %3 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %2, i32 0) - ret <8 x i16> %3 -} -; CHECK-LABEL: test_psra_4 -; CHECK-NOT: psraw -; CHECK: ret - - -declare <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16>, <8 x i16>) -declare <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16>, i32) -declare <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32>, <4 x i32>) -declare <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32>, i32) - diff --git a/test/CodeGen/X86/combine-sse41-intrinsics.ll b/test/CodeGen/X86/combine-sse41-intrinsics.ll index 254991aec094..1916883c201b 100644 --- a/test/CodeGen/X86/combine-sse41-intrinsics.ll +++ b/test/CodeGen/X86/combine-sse41-intrinsics.ll @@ -19,33 +19,6 @@ define <4 x float> @test_x86_sse41_blend_ps(<4 x float> %a0, <4 x float> %a1) { ; CHECK: ret -define <2 x double> @test_x86_sse41_blendv_pd(<2 x double> %a0, <2 x double> %a1) { - %1 = call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %a0, <2 x double> %a1, <2 x double> zeroinitializer) - ret <2 x double> %1 -} -; CHECK-LABEL: test_x86_sse41_blendv_pd -; CHECK-NOT: blendvpd -; CHECK: ret - - -define <4 x float> @test_x86_sse41_blendv_ps(<4 x float> %a0, <4 x float> %a1) { - %1 = call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %a0, <4 x float> %a1, <4 x float> zeroinitializer) - ret <4 x float> %1 -} -; CHECK-LABEL: test_x86_sse41_blendv_ps -; CHECK-NOT: blendvps -; CHECK: ret - - -define <16 x i8> @test_x86_sse41_pblendv_b(<16 x i8> %a0, <16 x i8> %a1) { - %1 = call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> zeroinitializer) - ret <16 x i8> %1 -} -; CHECK-LABEL: test_x86_sse41_pblendv_b -; CHECK-NOT: pblendvb -; CHECK: ret - - define <8 x i16> @test_x86_sse41_pblend_w(<8 x i16> %a0, <8 x i16> %a1) { %1 = call <8 x i16> @llvm.x86.sse41.pblendw(<8 x i16> %a0, <8 x i16> %a1, i32 0) ret <8 x i16> %1 @@ -75,39 +48,6 @@ define <4 x float> @test2_x86_sse41_blend_ps(<4 x float> %a0, <4 x float> %a1) { ; CHECK-NEXT: ret -define <2 x double> @test2_x86_sse41_blendv_pd(<2 x double> %a0, <2 x double> %a1) { - %Mask = bitcast <2 x i64> to <2 x double> - %1 = call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %a0, <2 x double> %a1, <2 x double> %Mask ) - ret <2 x double> %1 -} -; CHECK-LABEL: test2_x86_sse41_blendv_pd -; CHECK-NOT: blendvpd -; CHECK: movaps %xmm1, %xmm0 -; CHECK-NEXT: ret - - -define <4 x float> @test2_x86_sse41_blendv_ps(<4 x float> %a0, <4 x float> %a1) { - %Mask = bitcast <2 x i64> to <4 x float> - %1 = call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %a0, <4 x float> %a1, <4 x float> %Mask) - ret <4 x float> %1 -} -; CHECK-LABEL: test2_x86_sse41_blendv_ps -; CHECK-NOT: blendvps -; CHECK: movaps %xmm1, %xmm0 -; CHECK-NEXT: ret - - -define <16 x i8> @test2_x86_sse41_pblendv_b(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2) { - %Mask = bitcast <2 x i64> to <16 x i8> - %1 = call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %Mask) - ret <16 x i8> %1 -} -; CHECK-LABEL: test2_x86_sse41_pblendv_b -; CHECK-NOT: pblendvb -; CHECK: movaps %xmm1, %xmm0 -; CHECK-NEXT: ret - - define <8 x i16> @test2_x86_sse41_pblend_w(<8 x i16> %a0, <8 x i16> %a1) { %1 = call <8 x i16> @llvm.x86.sse41.pblendw(<8 x i16> %a0, <8 x i16> %a1, i32 -1) ret <8 x i16> %1 @@ -136,33 +76,6 @@ define <4 x float> @test3_x86_sse41_blend_ps(<4 x float> %a0) { ; CHECK: ret -define <2 x double> @test3_x86_sse41_blendv_pd(<2 x double> %a0, <2 x double> %a1) { - %1 = call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %a0, <2 x double> %a0, <2 x double> %a1 ) - ret <2 x double> %1 -} -; CHECK-LABEL: test3_x86_sse41_blendv_pd -; CHECK-NOT: blendvpd -; CHECK: ret - - -define <4 x float> @test3_x86_sse41_blendv_ps(<4 x float> %a0, <4 x float> %a1) { - %1 = call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %a0, <4 x float> %a0, <4 x float> %a1) - ret <4 x float> %1 -} -; CHECK-LABEL: test3_x86_sse41_blendv_ps -; CHECK-NOT: blendvps -; CHECK: ret - - -define <16 x i8> @test3_x86_sse41_pblendv_b(<16 x i8> %a0, <16 x i8> %a1) { - %1 = call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %a0, <16 x i8> %a0, <16 x i8> %a1) - ret <16 x i8> %1 -} -; CHECK-LABEL: test3_x86_sse41_pblendv_b -; CHECK-NOT: pblendvb -; CHECK: ret - - define <8 x i16> @test3_x86_sse41_pblend_w(<8 x i16> %a0) { %1 = call <8 x i16> @llvm.x86.sse41.pblendw(<8 x i16> %a0, <8 x i16> %a0, i32 7) ret <8 x i16> %1 @@ -174,9 +87,5 @@ define <8 x i16> @test3_x86_sse41_pblend_w(<8 x i16> %a0) { declare <2 x double> @llvm.x86.sse41.blendpd(<2 x double>, <2 x double>, i32) declare <4 x float> @llvm.x86.sse41.blendps(<4 x float>, <4 x float>, i32) -declare <2 x double> @llvm.x86.sse41.blendvpd(<2 x double>, <2 x double>, <2 x double>) -declare <4 x float> @llvm.x86.sse41.blendvps(<4 x float>, <4 x float>, <4 x float>) -declare <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8>, <16 x i8>, <16 x i8>) declare <8 x i16> @llvm.x86.sse41.pblendw(<8 x i16>, <8 x i16>, i32) -declare <8 x i16> @llvm.x86.sse41.phminposuw(<8 x i16>) diff --git a/test/CodeGen/X86/commute-two-addr.ll b/test/CodeGen/X86/commute-two-addr.ll index 5b01e2f4e90d..656c385e2bc7 100644 --- a/test/CodeGen/X86/commute-two-addr.ll +++ b/test/CodeGen/X86/commute-two-addr.ll @@ -39,7 +39,7 @@ define %0 @t3(i32 %lb, i8 zeroext %has_lb, i8 zeroext %lb_inclusive, i32 %ub, i8 entry: ; DARWIN-LABEL: t3: ; DARWIN: shlq $32, %rcx -; DARWIN-NEXT: leaq (%rax,%rcx), %rax +; DARWIN-NEXT: orq %rcx, %rax ; DARWIN-NEXT: shll $8 ; DARWIN-NOT: leaq %tmp21 = zext i32 %lb to i64 diff --git a/test/CodeGen/X86/constant-hoisting-and.ll b/test/CodeGen/X86/constant-hoisting-and.ll new file mode 100644 index 000000000000..611445f4a249 --- /dev/null +++ b/test/CodeGen/X86/constant-hoisting-and.ll @@ -0,0 +1,19 @@ +; RUN: llc < %s -O3 -march=x86-64 |FileCheck %s +define i64 @foo(i1 %z, i64 %data1, i64 %data2) +{ +; If constant 4294967294 is hoisted to a variable, then we won't be able to use +; the implicit zero extension of 32-bit operations to handle the AND. +entry: + %val1 = and i64 %data1, 4294967294 + br i1 %z, label %End, label %L_val2 + +; CHECK: andl $-2, {{.*}} +; CHECK: andl $-2, {{.*}} +L_val2: + %val2 = and i64 %data2, 4294967294 + br label %End + +End: + %p1 = phi i64 [%val1,%entry], [%val2,%L_val2] + ret i64 %p1 +} diff --git a/test/CodeGen/X86/constant-hoisting-cmp.ll b/test/CodeGen/X86/constant-hoisting-cmp.ll new file mode 100644 index 000000000000..4e9e49487287 --- /dev/null +++ b/test/CodeGen/X86/constant-hoisting-cmp.ll @@ -0,0 +1,25 @@ +; RUN: llc < %s -O3 -march=x86-64 |FileCheck %s +define i64 @foo(i64 %data1, i64 %data2, i64 %data3) +{ +; If constant 4294967295 is hoisted to a variable, then we won't be able to +; use a shift right by 32 to optimize the compare. +entry: + %val1 = add i64 %data3, 1 + %x = icmp ugt i64 %data1, 4294967295 + br i1 %x, label %End, label %L_val2 + +; CHECK: shrq $32, {{.*}} +; CHECK: shrq $32, {{.*}} +L_val2: + %val2 = add i64 %data3, 2 + %y = icmp ugt i64 %data2, 4294967295 + br i1 %y, label %End, label %L_val3 + +L_val3: + %val3 = add i64 %data3, 3 + br label %End + +End: + %p1 = phi i64 [%val1,%entry], [%val2,%L_val2], [%val3,%L_val3] + ret i64 %p1 +} diff --git a/test/CodeGen/X86/copysign-constant-magnitude.ll b/test/CodeGen/X86/copysign-constant-magnitude.ll index 537d6298ddf4..6c577a2cfcc7 100644 --- a/test/CodeGen/X86/copysign-constant-magnitude.ll +++ b/test/CodeGen/X86/copysign-constant-magnitude.ll @@ -5,13 +5,13 @@ target triple = "x86_64-apple-macosx10.10.0" define void @test_copysign_const_magnitude_d(double %X) { ; CHECK: [[SIGNMASK:L.+]]: -; CHECK-NEXT: .quad -9223372036854775808 ## double -0.000000e+00 -; CHECK-NEXT: .quad 0 ## double 0.000000e+00 +; CHECK-NEXT: .quad -9223372036854775808 ## double -0 +; CHECK-NEXT: .quad 0 ## double 0 ; CHECK: [[ZERO:L.+]]: ; CHECK-NEXT: .space 16 ; CHECK: [[ONE:L.+]]: -; CHECK-NEXT: .quad 4607182418800017408 ## double 1.000000e+00 -; CHECK-NEXT: .quad 0 ## double 0.000000e+00 +; CHECK-NEXT: .quad 4607182418800017408 ## double 1 +; CHECK-NEXT: .quad 0 ## double 0 ; CHECK-LABEL: test_copysign_const_magnitude_d: ; CHECK: id @@ -50,17 +50,17 @@ define void @test_copysign_const_magnitude_d(double %X) { define void @test_copysign_const_magnitude_f(float %X) { ; CHECK: [[SIGNMASK:L.+]]: -; CHECK-NEXT: .long 2147483648 ## float -0.000000e+00 -; CHECK-NEXT: .long 0 ## float 0.000000e+00 -; CHECK-NEXT: .long 0 ## float 0.000000e+00 -; CHECK-NEXT: .long 0 ## float 0.000000e+00 +; CHECK-NEXT: .long 2147483648 ## float -0 +; CHECK-NEXT: .long 0 ## float 0 +; CHECK-NEXT: .long 0 ## float 0 +; CHECK-NEXT: .long 0 ## float 0 ; CHECK: [[ZERO:L.+]]: ; CHECK-NEXT: .space 16 ; CHECK: [[ONE:L.+]]: -; CHECK-NEXT: .long 1065353216 ## float 1.000000e+00 -; CHECK-NEXT: .long 0 ## float 0.000000e+00 -; CHECK-NEXT: .long 0 ## float 0.000000e+00 -; CHECK-NEXT: .long 0 ## float 0.000000e+00 +; CHECK-NEXT: .long 1065353216 ## float 1 +; CHECK-NEXT: .long 0 ## float 0 +; CHECK-NEXT: .long 0 ## float 0 +; CHECK-NEXT: .long 0 ## float 0 ; CHECK-LABEL: test_copysign_const_magnitude_f: ; CHECK: id diff --git a/test/CodeGen/X86/cppeh-nounwind.ll b/test/CodeGen/X86/cppeh-nounwind.ll deleted file mode 100644 index d9bc001a92df..000000000000 --- a/test/CodeGen/X86/cppeh-nounwind.ll +++ /dev/null @@ -1,35 +0,0 @@ -; RUN: llc -mtriple=i686-pc-windows-msvc < %s | FileCheck %s - -; Sometimes invokes of nounwind functions make it through to CodeGen, especially -; at -O0, where Clang sometimes optimistically annotates functions as nounwind. -; WinEHPrepare ends up outlining functions, and emitting references to LSDA -; labels. Make sure we emit the LSDA in that case. - -declare i32 @__CxxFrameHandler3(...) -declare void @nounwind_func() nounwind -declare void @cleanup() - -define void @should_emit_tables() personality i32 (...)* @__CxxFrameHandler3 { -entry: - invoke void @nounwind_func() - to label %done unwind label %lpad - -done: - ret void - -lpad: - %vals = landingpad { i8*, i32 } - cleanup - call void @cleanup() - resume { i8*, i32 } %vals -} - -; CHECK: _should_emit_tables: -; CHECK: calll _nounwind_func -; CHECK: retl - -; CHECK: L__ehtable$should_emit_tables: - -; CHECK: ___ehhandler$should_emit_tables: -; CHECK: movl $L__ehtable$should_emit_tables, %eax -; CHECK: jmp ___CxxFrameHandler3 # TAILCALL diff --git a/test/CodeGen/X86/cxx_tlscc64.ll b/test/CodeGen/X86/cxx_tlscc64.ll new file mode 100644 index 000000000000..c229521cc9a4 --- /dev/null +++ b/test/CodeGen/X86/cxx_tlscc64.ll @@ -0,0 +1,71 @@ +; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-apple-darwin -enable-shrink-wrap=true | FileCheck --check-prefix=SHRINK %s +%struct.S = type { i8 } + +@sg = internal thread_local global %struct.S zeroinitializer, align 1 +@__dso_handle = external global i8 +@__tls_guard = internal thread_local unnamed_addr global i1 false + +declare void @_ZN1SC1Ev(%struct.S*) +declare void @_ZN1SD1Ev(%struct.S*) +declare i32 @_tlv_atexit(void (i8*)*, i8*, i8*) + +; Every GPR should be saved - except rdi, rax, and rsp +; CHECK-LABEL: _ZTW2sg +; CHECK: pushq %r11 +; CHECK: pushq %r10 +; CHECK: pushq %r9 +; CHECK: pushq %r8 +; CHECK: pushq %rsi +; CHECK: pushq %rdx +; CHECK: pushq %rcx +; CHECK: pushq %rbx +; CHECK: callq +; CHECK: jne +; CHECK: callq +; CHECK: tlv_atexit +; CHECK: callq +; CHECK: popq %rbx +; CHECK: popq %rcx +; CHECK: popq %rdx +; CHECK: popq %rsi +; CHECK: popq %r8 +; CHECK: popq %r9 +; CHECK: popq %r10 +; CHECK: popq %r11 +; SHRINK-LABEL: _ZTW2sg +; SHRINK: callq +; SHRINK: jne +; SHRINK: pushq %r11 +; SHRINK: pushq %r10 +; SHRINK: pushq %r9 +; SHRINK: pushq %r8 +; SHRINK: pushq %rsi +; SHRINK: pushq %rdx +; SHRINK: pushq %rcx +; SHRINK: pushq %rbx +; SHRINK: callq +; SHRINK: tlv_atexit +; SHRINK: popq %rbx +; SHRINK: popq %rcx +; SHRINK: popq %rdx +; SHRINK: popq %rsi +; SHRINK: popq %r8 +; SHRINK: popq %r9 +; SHRINK: popq %r10 +; SHRINK: popq %r11 +; SHRINK: LBB{{.*}}: +; SHRINK: callq +define cxx_fast_tlscc nonnull %struct.S* @_ZTW2sg() { + %.b.i = load i1, i1* @__tls_guard, align 1 + br i1 %.b.i, label %__tls_init.exit, label %init.i + +init.i: + store i1 true, i1* @__tls_guard, align 1 + tail call void @_ZN1SC1Ev(%struct.S* nonnull @sg) #2 + %1 = tail call i32 @_tlv_atexit(void (i8*)* nonnull bitcast (void (%struct.S*)* @_ZN1SD1Ev to void (i8*)*), i8* nonnull getelementptr inbounds (%struct.S, %struct.S* @sg, i64 0, i32 0), i8* nonnull @__dso_handle) #2 + br label %__tls_init.exit + +__tls_init.exit: + ret %struct.S* @sg +} diff --git a/test/CodeGen/X86/dag-fmf-cse.ll b/test/CodeGen/X86/dag-fmf-cse.ll new file mode 100644 index 000000000000..ac8c5000aba4 --- /dev/null +++ b/test/CodeGen/X86/dag-fmf-cse.ll @@ -0,0 +1,22 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=fma -enable-unsafe-fp-math -enable-fmf-dag=1 | FileCheck %s + +; If fast-math-flags are propagated correctly, the mul1 expression +; should be recognized as a factor in the last fsub, so we should +; see a mul and add, not a mul and fma: +; a * b - (-a * b) ---> (a * b) + (a * b) + +define float @fmf_should_not_break_cse(float %a, float %b) { +; CHECK-LABEL: fmf_should_not_break_cse: +; CHECK: # BB#0: +; CHECK-NEXT: vmulss %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vaddss %xmm0, %xmm0, %xmm0 +; CHECK-NEXT: retq + + %mul1 = fmul fast float %a, %b + %nega = fsub fast float 0.0, %a + %mul2 = fmul fast float %nega, %b + %abx2 = fsub fast float %mul1, %mul2 + ret float %abx2 +} + diff --git a/test/CodeGen/X86/dag-merge-fast-accesses.ll b/test/CodeGen/X86/dag-merge-fast-accesses.ll new file mode 100644 index 000000000000..867881d83d3f --- /dev/null +++ b/test/CodeGen/X86/dag-merge-fast-accesses.ll @@ -0,0 +1,90 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-slow-unaligned-mem-16 | FileCheck %s --check-prefix=FAST +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+slow-unaligned-mem-16 | FileCheck %s --check-prefix=SLOW + +; Verify that the DAGCombiner is creating unaligned 16-byte loads and stores +; if and only if those are fast. + +define void @merge_const_vec_store(i64* %ptr) { +; FAST-LABEL: merge_const_vec_store: +; FAST: # BB#0: +; FAST-NEXT: xorps %xmm0, %xmm0 +; FAST-NEXT: movups %xmm0, (%rdi) +; FAST-NEXT: retq +; +; SLOW-LABEL: merge_const_vec_store: +; SLOW: # BB#0: +; SLOW-NEXT: movq $0, (%rdi) +; SLOW-NEXT: movq $0, 8(%rdi) +; SLOW-NEXT: retq + + %idx0 = getelementptr i64, i64* %ptr, i64 0 + %idx1 = getelementptr i64, i64* %ptr, i64 1 + + store i64 0, i64* %idx0, align 8 + store i64 0, i64* %idx1, align 8 + ret void +} + + +define void @merge_vec_element_store(<4 x double> %v, double* %ptr) { +; FAST-LABEL: merge_vec_element_store: +; FAST: # BB#0: +; FAST-NEXT: movups %xmm0, (%rdi) +; FAST-NEXT: retq +; +; SLOW-LABEL: merge_vec_element_store: +; SLOW: # BB#0: +; SLOW-NEXT: movlpd %xmm0, (%rdi) +; SLOW-NEXT: movhpd %xmm0, 8(%rdi) +; SLOW-NEXT: retq + + %vecext0 = extractelement <4 x double> %v, i32 0 + %vecext1 = extractelement <4 x double> %v, i32 1 + + %idx0 = getelementptr double, double* %ptr, i64 0 + %idx1 = getelementptr double, double* %ptr, i64 1 + + store double %vecext0, double* %idx0, align 8 + store double %vecext1, double* %idx1, align 8 + ret void +} + + +;; TODO: FAST *should* be: +;; movups (%rdi), %xmm0 +;; movups %xmm0, 40(%rdi) +;; ..but is not currently. See the UseAA FIXME in DAGCombiner.cpp +;; visitSTORE. + +define void @merge_vec_load_and_stores(i64 *%ptr) { +; FAST-LABEL: merge_vec_load_and_stores: +; FAST: # BB#0: +; FAST-NEXT: movq (%rdi), %rax +; FAST-NEXT: movq 8(%rdi), %rcx +; FAST-NEXT: movq %rax, 40(%rdi) +; FAST-NEXT: movq %rcx, 48(%rdi) +; FAST-NEXT: retq +; +; SLOW-LABEL: merge_vec_load_and_stores: +; SLOW: # BB#0: +; SLOW-NEXT: movq (%rdi), %rax +; SLOW-NEXT: movq 8(%rdi), %rcx +; SLOW-NEXT: movq %rax, 40(%rdi) +; SLOW-NEXT: movq %rcx, 48(%rdi) +; SLOW-NEXT: retq + + %idx0 = getelementptr i64, i64* %ptr, i64 0 + %idx1 = getelementptr i64, i64* %ptr, i64 1 + + %ld0 = load i64, i64* %idx0, align 4 + %ld1 = load i64, i64* %idx1, align 4 + + %idx4 = getelementptr i64, i64* %ptr, i64 5 + %idx5 = getelementptr i64, i64* %ptr, i64 6 + + store i64 %ld0, i64* %idx4, align 4 + store i64 %ld1, i64* %idx5, align 4 + ret void +} + diff --git a/test/CodeGen/X86/darwin-tls.ll b/test/CodeGen/X86/darwin-tls.ll new file mode 100644 index 000000000000..ca9a998ccc75 --- /dev/null +++ b/test/CodeGen/X86/darwin-tls.ll @@ -0,0 +1,28 @@ +; RUN: llc < %s -mtriple x86_64-apple-darwin | FileCheck %s + +@a = thread_local global i32 4, align 4 + +define i32 @f2(i32 %p1, i32 %p2, i32 %p3, i32 %p4, i32 %p5) { +entry: +; Parameters are in %edi, %esi, %edx, %ecx, %r8d, there is no need to save +; these parameters except the one in %edi, before making the TLS call. +; %edi is used to pass parameter to the TLS call. +; CHECK-NOT: movl %r8d +; CHECK-NOT: movl %ecx +; CHECK-NOT: movl %edx +; CHECK-NOT: movl %esi +; CHECK: movq {{.*}}TLVP{{.*}}, %rdi +; CHECK-NEXT: callq +; CHECK-NEXT: movl (%rax), +; CHECK-NOT: movl {{.*}}, %esi +; CHECK-NOT: movl {{.*}}, %edx +; CHECK-NOT: movl {{.*}}, %ecx +; CHECK-NOT: movl {{.*}}, %r8d +; CHECK: callq + %0 = load i32, i32* @a, align 4 + %call = tail call i32 @f3(i32 %p1, i32 %p2, i32 %p3, i32 %p4, i32 %p5) + %add = add nsw i32 %call, %0 + ret i32 %add +} + +declare i32 @f3(i32, i32, i32, i32, i32) diff --git a/test/CodeGen/X86/dbg-changes-codegen-branch-folding.ll b/test/CodeGen/X86/dbg-changes-codegen-branch-folding.ll index 20d0129c3e89..54bd48926834 100644 --- a/test/CodeGen/X86/dbg-changes-codegen-branch-folding.ll +++ b/test/CodeGen/X86/dbg-changes-codegen-branch-folding.ll @@ -48,7 +48,7 @@ @.str2 = private unnamed_addr constant [2 x i8] c"-\00", align 1 ; Function Attrs: uwtable -define void @_Z3barii(i32 %param1, i32 %param2) #0 { +define void @_Z3barii(i32 %param1, i32 %param2) #0 !dbg !24 { entry: %var1 = alloca %struct.AAA3, align 1 %var2 = alloca %struct.AAA3, align 1 @@ -113,7 +113,7 @@ attributes #2 = { nounwind readnone } !llvm.module.flags = !{!44, !45} !llvm.ident = !{!46} -!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !23, globals: !2, imports: !2) +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !3, subprograms: !23, globals: !2, imports: !2) !1 = !DIFile(filename: "dbg-changes-codegen-branch-folding.cpp", directory: "/tmp/dbginfo") !2 = !{} !3 = !{!4} @@ -137,26 +137,26 @@ attributes #2 = { nounwind readnone } !21 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !22) !22 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !"_ZTS4AAA3") !23 = !{!24, !35, !40} -!24 = !DISubprogram(name: "bar", linkageName: "_Z3barii", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !1, scope: !25, type: !26, function: void (i32, i32)* @_Z3barii, variables: !29) +!24 = distinct !DISubprogram(name: "bar", linkageName: "_Z3barii", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 11, file: !1, scope: !25, type: !26, variables: !29) !25 = !DIFile(filename: "dbg-changes-codegen-branch-folding.cpp", directory: "/tmp/dbginfo") !26 = !DISubroutineType(types: !27) !27 = !{null, !28, !28} !28 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !29 = !{!30, !31, !32, !33, !34} -!30 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "param1", line: 11, arg: 1, scope: !24, file: !25, type: !28) -!31 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "param2", line: 11, arg: 2, scope: !24, file: !25, type: !28) -!32 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "temp", line: 12, scope: !24, file: !25, type: !15) -!33 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "var1", line: 17, scope: !24, file: !25, type: !"_ZTS4AAA3") -!34 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "var2", line: 18, scope: !24, file: !25, type: !"_ZTS4AAA3") -!35 = !DISubprogram(name: "operator=", linkageName: "_ZN4AAA3aSEPKc", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !1, scope: !"_ZTS4AAA3", type: !12, declaration: !17, variables: !36) +!30 = !DILocalVariable(name: "param1", line: 11, arg: 1, scope: !24, file: !25, type: !28) +!31 = !DILocalVariable(name: "param2", line: 11, arg: 2, scope: !24, file: !25, type: !28) +!32 = !DILocalVariable(name: "temp", line: 12, scope: !24, file: !25, type: !15) +!33 = !DILocalVariable(name: "var1", line: 17, scope: !24, file: !25, type: !"_ZTS4AAA3") +!34 = !DILocalVariable(name: "var2", line: 18, scope: !24, file: !25, type: !"_ZTS4AAA3") +!35 = distinct !DISubprogram(name: "operator=", linkageName: "_ZN4AAA3aSEPKc", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !1, scope: !"_ZTS4AAA3", type: !12, declaration: !17, variables: !36) !36 = !{!37, !39} -!37 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, type: !38) +!37 = !DILocalVariable(name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, type: !38) !38 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !"_ZTS4AAA3") -!39 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 6, arg: 2, scope: !35, file: !25, type: !15) -!40 = !DISubprogram(name: "AAA3", linkageName: "_ZN4AAA3C2EPKc", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !1, scope: !"_ZTS4AAA3", type: !12, declaration: !11, variables: !41) +!39 = !DILocalVariable(name: "value", line: 6, arg: 2, scope: !35, file: !25, type: !15) +!40 = distinct !DISubprogram(name: "AAA3", linkageName: "_ZN4AAA3C2EPKc", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 5, file: !1, scope: !"_ZTS4AAA3", type: !12, declaration: !11, variables: !41) !41 = !{!42, !43} -!42 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !40, type: !38) -!43 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 5, arg: 2, scope: !40, file: !25, type: !15) +!42 = !DILocalVariable(name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !40, type: !38) +!43 = !DILocalVariable(name: "value", line: 5, arg: 2, scope: !40, file: !25, type: !15) !44 = !{i32 2, !"Dwarf Version", i32 4} !45 = !{i32 2, !"Debug Info Version", i32 3} !46 = !{!"clang version 3.5.0 "} @@ -169,36 +169,36 @@ attributes #2 = { nounwind readnone } !53 = distinct !DILexicalBlock(line: 14, column: 0, file: !1, scope: !51) !54 = !DILocation(line: 16, scope: !53) !55 = !DILocation(line: 17, scope: !24) -!56 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !40, type: !38) +!56 = !DILocalVariable(name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !40, type: !38) !57 = !DILocation(line: 0, scope: !40, inlinedAt: !55) !58 = !{i8* getelementptr inbounds ([1 x i8], [1 x i8]* @.str, i64 0, i64 0)} -!59 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 5, arg: 2, scope: !40, file: !25, type: !15) +!59 = !DILocalVariable(name: "value", line: 5, arg: 2, scope: !40, file: !25, type: !15) !60 = !DILocation(line: 5, scope: !40, inlinedAt: !55) !61 = !DILocation(line: 5, scope: !62, inlinedAt: !55) !62 = distinct !DILexicalBlock(line: 5, column: 0, file: !1, scope: !40) !63 = !DILocation(line: 18, scope: !24) -!64 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !40, type: !38) +!64 = !DILocalVariable(name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !40, type: !38) !65 = !DILocation(line: 0, scope: !40, inlinedAt: !63) -!66 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 5, arg: 2, scope: !40, file: !25, type: !15) +!66 = !DILocalVariable(name: "value", line: 5, arg: 2, scope: !40, file: !25, type: !15) !67 = !DILocation(line: 5, scope: !40, inlinedAt: !63) !68 = !DILocation(line: 5, scope: !62, inlinedAt: !63) !69 = !DILocation(line: 20, scope: !70) !70 = distinct !DILexicalBlock(line: 20, column: 0, file: !1, scope: !24) -!71 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, type: !38) +!71 = !DILocalVariable(name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, type: !38) !72 = !DILocation(line: 21, scope: !70) !73 = !DILocation(line: 0, scope: !35, inlinedAt: !72) !74 = !{i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str1, i64 0, i64 0)} -!75 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 6, arg: 2, scope: !35, file: !25, type: !15) +!75 = !DILocalVariable(name: "value", line: 6, arg: 2, scope: !35, file: !25, type: !15) !76 = !DILocation(line: 6, scope: !35, inlinedAt: !72) -!77 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, type: !38) +!77 = !DILocalVariable(name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, type: !38) !78 = !DILocation(line: 23, scope: !70) !79 = !DILocation(line: 0, scope: !35, inlinedAt: !78) !80 = !{i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str2, i64 0, i64 0)} -!81 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 6, arg: 2, scope: !35, file: !25, type: !15) +!81 = !DILocalVariable(name: "value", line: 6, arg: 2, scope: !35, file: !25, type: !15) !82 = !DILocation(line: 6, scope: !35, inlinedAt: !78) -!83 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, type: !38) +!83 = !DILocalVariable(name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !35, type: !38) !84 = !DILocation(line: 24, scope: !24) !85 = !DILocation(line: 0, scope: !35, inlinedAt: !84) -!86 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "value", line: 6, arg: 2, scope: !35, file: !25, type: !15) +!86 = !DILocalVariable(name: "value", line: 6, arg: 2, scope: !35, file: !25, type: !15) !87 = !DILocation(line: 6, scope: !35, inlinedAt: !84) !88 = !DILocation(line: 25, scope: !24) diff --git a/test/CodeGen/X86/dbg-changes-codegen.ll b/test/CodeGen/X86/dbg-changes-codegen.ll index b15e4bd4bf2d..bee86b4617c7 100644 --- a/test/CodeGen/X86/dbg-changes-codegen.ll +++ b/test/CodeGen/X86/dbg-changes-codegen.ll @@ -44,7 +44,7 @@ define zeroext i1 @_ZN3Foo3batEv(%struct.Foo* %this) #0 align 2 { entry: %0 = load %struct.Foo*, %struct.Foo** @pfoo, align 8 - tail call void @llvm.dbg.value(metadata %struct.Foo* %0, i64 0, metadata !62, metadata !DIExpression()), !dbg !DILocation(scope: !DISubprogram()) + tail call void @llvm.dbg.value(metadata %struct.Foo* %0, i64 0, metadata !62, metadata !DIExpression()), !dbg !DILocation(scope: !1) %cmp.i = icmp eq %struct.Foo* %0, %this ret i1 %cmp.i } @@ -53,7 +53,7 @@ entry: define void @_Z3bazv() #1 { entry: %0 = load %struct.Wibble*, %struct.Wibble** @wibble1, align 8 - tail call void @llvm.dbg.value(metadata %struct.Flibble* undef, i64 0, metadata !65, metadata !DIExpression()), !dbg !DILocation(scope: !DISubprogram()) + tail call void @llvm.dbg.value(metadata %struct.Flibble* undef, i64 0, metadata !65, metadata !DIExpression()), !dbg !DILocation(scope: !1) %1 = load %struct.Wibble*, %struct.Wibble** @wibble2, align 8 %cmp.i = icmp ugt %struct.Wibble* %1, %0 br i1 %cmp.i, label %if.then.i, label %_ZN7Flibble3barEP6Wibble.exit @@ -75,9 +75,10 @@ attributes #0 = { nounwind readonly uwtable "less-precise-fpmad"="false" "no-fra attributes #1 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #2 = { nounwind readnone } +!1 = distinct !DISubprogram() !17 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: null) !45 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null) -!62 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "arg", line: 4, arg: 2, scope: !DISubprogram(), type: !17) +!62 = !DILocalVariable(name: "arg", line: 4, arg: 2, scope: !1, type: !17) !64 = !{%struct.Flibble* undef} -!65 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", line: 13, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !DISubprogram(), type: !45) +!65 = !DILocalVariable(name: "this", line: 13, arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !1, type: !45) diff --git a/test/CodeGen/X86/dbg-combine.ll b/test/CodeGen/X86/dbg-combine.ll index 5eb2ea9df513..3e78c316a06f 100644 --- a/test/CodeGen/X86/dbg-combine.ll +++ b/test/CodeGen/X86/dbg-combine.ll @@ -24,7 +24,7 @@ ; ModuleID = 'dbg-combine.c' ; Function Attrs: nounwind uwtable -define i32 @foo() #0 { +define i32 @foo() #0 !dbg !4 { entry: %elems = alloca i32, align 4 %saved_stack = alloca i8* @@ -74,11 +74,11 @@ attributes #2 = { nounwind } !llvm.module.flags = !{!9, !10} !llvm.ident = !{!11} -!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0 (trunk 227074)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2) +!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.7.0 (trunk 227074)", isOptimized: false, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !2, imports: !2) !1 = !DIFile(filename: "dbg-combine.c", directory: "/home/probinson/projects/scratch") !2 = !{} !3 = !{!4} -!4 = !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, function: i32 ()* @foo, variables: !2) +!4 = distinct !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, isOptimized: false, scopeLine: 2, file: !1, scope: !5, type: !6, variables: !2) !5 = !DIFile(filename: "dbg-combine.c", directory: "/home/probinson/projects/scratch") !6 = !DISubroutineType(types: !7) !7 = !{!8} @@ -86,12 +86,12 @@ attributes #2 = { nounwind } !9 = !{i32 2, !"Dwarf Version", i32 4} !10 = !{i32 2, !"Debug Info Version", i32 3} !11 = !{!"clang version 3.7.0 (trunk 227074)"} -!12 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "elems", line: 3, scope: !4, file: !5, type: !8) +!12 = !DILocalVariable(name: "elems", line: 3, scope: !4, file: !5, type: !8) !13 = !DIExpression() !14 = !DILocation(line: 3, column: 8, scope: !4) !15 = !DILocation(line: 4, column: 15, scope: !4) !16 = !DILocation(line: 4, column: 4, scope: !4) -!17 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "array1", line: 4, scope: !4, file: !5, type: !18) +!17 = !DILocalVariable(name: "array1", line: 4, scope: !4, file: !5, type: !18) !18 = !DICompositeType(tag: DW_TAG_array_type, align: 32, baseType: !8, elements: !19) !19 = !{!20} !20 = !DISubrange(count: -1) @@ -105,7 +105,7 @@ attributes #2 = { nounwind } !28 = !DILocation(line: 7, column: 13, scope: !4) !29 = !DILocation(line: 8, column: 15, scope: !4) !30 = !DILocation(line: 8, column: 4, scope: !4) -!31 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "array2", line: 8, scope: !4, file: !5, type: !18) +!31 = !DILocalVariable(name: "array2", line: 8, scope: !4, file: !5, type: !18) !32 = !DILocation(line: 8, column: 8, scope: !4) !33 = !DILocation(line: 9, column: 4, scope: !4) !34 = !DILocation(line: 9, column: 13, scope: !4) diff --git a/test/CodeGen/X86/debugloc-argsize.ll b/test/CodeGen/X86/debugloc-argsize.ll new file mode 100644 index 000000000000..0283154abab2 --- /dev/null +++ b/test/CodeGen/X86/debugloc-argsize.ll @@ -0,0 +1,58 @@ +; RUN: llc < %s -mtriple=i686-pc-linux | FileCheck %s + +; CHECK-LABEL: _Z3foov: +; CHECK: .loc 1 4 3 prologue_end +; CHECK: .cfi_escape 0x2e, 0x10 +define void @_Z3foov() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) !dbg !4 { +entry: + tail call void @_Z3bariii(i32 0, i32 1, i32 2) #1, !dbg !10 + invoke void @_Z3bariii(i32 4, i32 5, i32 6) #1 + to label %try.cont unwind label %lpad, !dbg !11 + +lpad: ; preds = %entry + %0 = landingpad { i8*, i32 } + catch i8* null, !dbg !13 + %1 = extractvalue { i8*, i32 } %0, 0, !dbg !13 + %2 = tail call i8* @__cxa_begin_catch(i8* %1) #2, !dbg !14 + tail call void @__cxa_end_catch(), !dbg !15 + br label %try.cont, !dbg !15 + +try.cont: ; preds = %entry, %lpad + ret void, !dbg !17 +} + +; Function Attrs: optsize +declare void @_Z3bariii(i32, i32, i32) #0 + +declare i32 @__gxx_personality_v0(...) + +declare i8* @__cxa_begin_catch(i8*) + +declare void @__cxa_end_catch() + +attributes #0 = { optsize "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { optsize } +attributes #2 = { nounwind } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!7, !8} +!llvm.ident = !{!9} + +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 249520)", isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !2, subprograms: !3) +!1 = !DIFile(filename: "foo.cpp", directory: "foo") +!2 = !{} +!3 = !{!4} +!4 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, variables: !2) +!5 = !DISubroutineType(types: !6) +!6 = !{null} +!7 = !{i32 2, !"Dwarf Version", i32 4} +!8 = !{i32 2, !"Debug Info Version", i32 3} +!9 = !{!"clang version 3.8.0 (trunk 249520)"} +!10 = !DILocation(line: 4, column: 3, scope: !4) +!11 = !DILocation(line: 6, column: 5, scope: !12) +!12 = distinct !DILexicalBlock(scope: !4, file: !1, line: 5, column: 7) +!13 = !DILocation(line: 10, column: 1, scope: !12) +!14 = !DILocation(line: 7, column: 3, scope: !12) +!15 = !DILocation(line: 9, column: 3, scope: !16) +!16 = distinct !DILexicalBlock(scope: !4, file: !1, line: 7, column: 17) +!17 = !DILocation(line: 10, column: 1, scope: !4) diff --git a/test/CodeGen/X86/divide-by-constant.ll b/test/CodeGen/X86/divide-by-constant.ll index fd07a3f55100..9543d6c4d749 100644 --- a/test/CodeGen/X86/divide-by-constant.ll +++ b/test/CodeGen/X86/divide-by-constant.ll @@ -94,3 +94,35 @@ define i8 @test9(i8 %x) nounwind { ; CHECK: shrl $11 ; CHECK: ret } + +define i32 @testsize1(i32 %x) minsize nounwind { +entry: + %div = sdiv i32 %x, 32 + ret i32 %div +; CHECK-LABEL: testsize1: +; CHECK: divl +} + +define i32 @testsize2(i32 %x) minsize nounwind { +entry: + %div = sdiv i32 %x, 33 + ret i32 %div +; CHECK-LABEL: testsize2: +; CHECK: divl +} + +define i32 @testsize3(i32 %x) minsize nounwind { +entry: + %div = udiv i32 %x, 32 + ret i32 %div +; CHECK-LABEL: testsize3: +; CHECK: shrl +} + +define i32 @testsize4(i32 %x) minsize nounwind { +entry: + %div = udiv i32 %x, 33 + ret i32 %div +; CHECK-LABEL: testsize4: +; CHECK: divl +} diff --git a/test/CodeGen/X86/dllexport-x86_64.ll b/test/CodeGen/X86/dllexport-x86_64.ll index bb5e92f98c7d..58e25f923971 100644 --- a/test/CodeGen/X86/dllexport-x86_64.ll +++ b/test/CodeGen/X86/dllexport-x86_64.ll @@ -53,22 +53,22 @@ define weak_odr dllexport void @weak1() { ; CHECK: .globl alias ; CHECK: alias = notExported -@alias = dllexport alias void()* @notExported +@alias = dllexport alias void(), void()* @notExported ; CHECK: .globl alias2 ; CHECK: alias2 = f1 -@alias2 = dllexport alias void()* @f1 +@alias2 = dllexport alias void(), void()* @f1 ; CHECK: .globl alias3 ; CHECK: alias3 = notExported -@alias3 = dllexport alias void()* @notExported +@alias3 = dllexport alias void(), void()* @notExported ; CHECK: .weak weak_alias ; CHECK: weak_alias = f1 -@weak_alias = weak_odr dllexport alias void()* @f1 +@weak_alias = weak_odr dllexport alias void(), void()* @f1 @blob = global [6 x i8] c"\B8*\00\00\00\C3", section ".text", align 16 -@blob_alias = dllexport alias bitcast ([6 x i8]* @blob to i32 ()*) +@blob_alias = dllexport alias i32 (), bitcast ([6 x i8]* @blob to i32 ()*) ; CHECK: .section .drectve ; WIN32: /EXPORT:f1 diff --git a/test/CodeGen/X86/dllexport.ll b/test/CodeGen/X86/dllexport.ll index 915567de5bf7..cde0955410b7 100644 --- a/test/CodeGen/X86/dllexport.ll +++ b/test/CodeGen/X86/dllexport.ll @@ -74,19 +74,19 @@ define weak_odr dllexport void @weak1() { ; CHECK: .globl _alias ; CHECK: _alias = _notExported -@alias = dllexport alias void()* @notExported +@alias = dllexport alias void(), void()* @notExported ; CHECK: .globl _alias2 ; CHECK: _alias2 = _f1 -@alias2 = dllexport alias void()* @f1 +@alias2 = dllexport alias void(), void()* @f1 ; CHECK: .globl _alias3 ; CHECK: _alias3 = _notExported -@alias3 = dllexport alias void()* @notExported +@alias3 = dllexport alias void(), void()* @notExported ; CHECK: .weak _weak_alias ; CHECK: _weak_alias = _f1 -@weak_alias = weak_odr dllexport alias void()* @f1 +@weak_alias = weak_odr dllexport alias void(), void()* @f1 ; CHECK: .section .drectve ; CHECK-CL-NOT: not_exported diff --git a/test/CodeGen/X86/dwarf-comp-dir.ll b/test/CodeGen/X86/dwarf-comp-dir.ll index 27b8b1552ec1..31d2724aade3 100644 --- a/test/CodeGen/X86/dwarf-comp-dir.ll +++ b/test/CodeGen/X86/dwarf-comp-dir.ll @@ -7,7 +7,7 @@ target triple = "x86_64-unknown-linux-gnu" !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!5} -!0 = !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 143523)", isOptimized: true, emissionKind: 0, file: !4, enums: !2, retainedTypes: !7, subprograms: !2, globals: !2) +!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 143523)", isOptimized: true, emissionKind: 0, file: !4, enums: !2, retainedTypes: !7, subprograms: !2, globals: !2) !2 = !{} !3 = !DIFile(filename: "empty.c", directory: "/home/nlewycky") !4 = !DIFile(filename: "empty.c", directory: "/home/nlewycky") diff --git a/test/CodeGen/X86/dynamic-allocas-VLAs.ll b/test/CodeGen/X86/dynamic-allocas-VLAs.ll index 2925f243b0e3..b0334d6a63ef 100644 --- a/test/CodeGen/X86/dynamic-allocas-VLAs.ll +++ b/test/CodeGen/X86/dynamic-allocas-VLAs.ll @@ -1,5 +1,5 @@ ; RUN: llc < %s -mcpu=generic -march=x86-64 -mattr=+avx -mtriple=i686-apple-darwin10 | FileCheck %s -; RUN: llc < %s -mcpu=generic -force-align-stack -stack-alignment=32 -march=x86-64 -mattr=+avx -mtriple=i686-apple-darwin10 | FileCheck %s -check-prefix=FORCE-ALIGN +; RUN: llc < %s -mcpu=generic -stackrealign -stack-alignment=32 -march=x86-64 -mattr=+avx -mtriple=i686-apple-darwin10 | FileCheck %s -check-prefix=FORCE-ALIGN ; rdar://11496434 ; no VLAs or dynamic alignment diff --git a/test/CodeGen/X86/eh-null-personality.ll b/test/CodeGen/X86/eh-null-personality.ll new file mode 100644 index 000000000000..536f060db8d9 --- /dev/null +++ b/test/CodeGen/X86/eh-null-personality.ll @@ -0,0 +1,25 @@ +; RUN: llc -mtriple=x86_64-linux < %s | FileCheck %s + +; We should treat non-Function personalities as the unknown personality, which +; is usually Itanium. + +declare void @g() +declare void @terminate(i8*) + +define void @f() personality i8* null { + invoke void @g() + to label %ret unwind label %lpad +ret: + ret void +lpad: + %vals = landingpad { i8*, i32 } catch i8* null + %ptr = extractvalue { i8*, i32 } %vals, 0 + call void @terminate(i8* %ptr) + unreachable +} + +; CHECK: f: +; CHECK: callq g +; CHECK: retq +; CHECK: movq %rax, %rdi +; CHECK: callq terminate diff --git a/test/CodeGen/X86/eh_frame.ll b/test/CodeGen/X86/eh_frame.ll index 3b792b235cb5..0472e773df56 100644 --- a/test/CodeGen/X86/eh_frame.ll +++ b/test/CodeGen/X86/eh_frame.ll @@ -7,8 +7,8 @@ @bar1 = constant i8* bitcast (i32* @foo to i8*), section "my_bar1", align 8 -; STATIC: .section .eh_frame,"a",@progbits +; STATIC: .section .eh_frame,"a",@unwind ; STATIC: .section my_bar1,"a",@progbits -; PIC: .section .eh_frame,"a",@progbits +; PIC: .section .eh_frame,"a",@unwind ; PIC: .section my_bar1,"aw",@progbits diff --git a/test/CodeGen/X86/emutls-pic.ll b/test/CodeGen/X86/emutls-pic.ll new file mode 100644 index 000000000000..11676aff1892 --- /dev/null +++ b/test/CodeGen/X86/emutls-pic.ll @@ -0,0 +1,168 @@ +; RUN: llc < %s -emulated-tls -march=x86 -mtriple=i386-linux-gnu -relocation-model=pic | FileCheck -check-prefix=X32 %s +; RUN: llc < %s -emulated-tls -march=x86-64 -mtriple=x86_64-linux-gnu -relocation-model=pic | FileCheck -check-prefix=X64 %s +; RUN: llc < %s -emulated-tls -march=x86 -mtriple=i386-linux-android -relocation-model=pic | FileCheck -check-prefix=X32 %s +; RUN: llc < %s -emulated-tls -march=x86-64 -mtriple=x86_64-linux-android -relocation-model=pic | FileCheck -check-prefix=X64 %s + +; Use my_emutls_get_address like __emutls_get_address. +@my_emutls_v_xyz = external global i8*, align 4 +declare i8* @my_emutls_get_address(i8*) + +define i32 @my_get_xyz() { +; X32-LABEL: my_get_xyz: +; X32: movl my_emutls_v_xyz@GOT(%ebx), %eax +; X32-NEXT: movl %eax, (%esp) +; X32-NEXT: calll my_emutls_get_address@PLT +; X64-LABEL: my_get_xyz: +; X64: movq my_emutls_v_xyz@GOTPCREL(%rip), %rdi +; X64-NEXT: callq my_emutls_get_address@PLT +; X64-NEXT: movl (%rax), %eax + +entry: + %call = call i8* @my_emutls_get_address(i8* bitcast (i8** @my_emutls_v_xyz to i8*)) + %0 = bitcast i8* %call to i32* + %1 = load i32, i32* %0, align 4 + ret i32 %1 +} + +@i = thread_local global i32 15 +@j = internal thread_local global i32 42 +@k = internal thread_local global i32 0, align 8 + +define i32 @f1() { +entry: + %tmp1 = load i32, i32* @i + ret i32 %tmp1 +} + +; X32-LABEL: f1: +; X32: movl __emutls_v.i@GOT(%ebx), %eax +; X32-NEXT: movl %eax, (%esp) +; X32-NEXT: calll __emutls_get_address@PLT +; X64-LABEL: f1: +; X64: movq __emutls_v.i@GOTPCREL(%rip), %rdi +; X64-NEXT: callq __emutls_get_address@PLT +; X64-NEXT: movl (%rax), %eax + +@i2 = external thread_local global i32 + +define i32* @f2() { +entry: + ret i32* @i +} + +; X32-LABEL: f2: +; X64-LABEL: f2: + + +define i32 @f3() { +entry: + %tmp1 = load i32, i32* @i ; [#uses=1] + ret i32 %tmp1 +} + +; X32-LABEL: f3: +; X64-LABEL: f3: + + +define i32* @f4() nounwind { +entry: + ret i32* @i +} + +; X32-LABEL: f4: +; X64-LABEL: f4: + + +define i32 @f5() nounwind { +entry: + %0 = load i32, i32* @j, align 4 + %1 = load i32, i32* @k, align 4 + %add = add nsw i32 %0, %1 + ret i32 %add +} + +; X32-LABEL: f5: +; X32: movl __emutls_v.j@GOT(%ebx), %eax +; X32-NEXT: movl %eax, (%esp) +; X32-NEXT: calll __emutls_get_address@PLT +; X32-NEXT: movl (%eax), %esi +; X32-NEXT: movl __emutls_v.k@GOT(%ebx), %eax +; X32-NEXT: movl %eax, (%esp) +; X32-NEXT: calll __emutls_get_address@PLT +; X32-NEXT: addl (%eax), %esi +; X32-NEXT: movl %esi, %eax + +; X64-LABEL: f5: +; X64: movq __emutls_v.j@GOTPCREL(%rip), %rdi +; X64-NEXT: callq __emutls_get_address@PLT +; X64-NEXT: movl (%rax), %ebx +; X64-NEXT: movq __emutls_v.k@GOTPCREL(%rip), %rdi +; X64-NEXT: callq __emutls_get_address@PLT +; X64-NEXT: addl (%rax), %ebx +; X64-NEXT: movl %ebx, %eax + +;;;;; 32-bit targets + +; X32: .data +; X32-LABEL: __emutls_v.i: +; X32-NEXT: .long 4 +; X32-NEXT: .long 4 +; X32-NEXT: .long 0 +; X32-NEXT: .long __emutls_t.i + +; X32: .section .rodata, +; X32-LABEL: __emutls_t.i: +; X32-NEXT: .long 15 + +; X32: .data +; X32-LABEL: __emutls_v.j: +; X32-NEXT: .long 4 +; X32-NEXT: .long 4 +; X32-NEXT: .long 0 +; X32-NEXT: .long __emutls_t.j + +; X32: .section .rodata, +; X32-LABEL: __emutls_t.j: +; X32-NEXT: .long 42 + +; X32: .data +; X32-LABEL: __emutls_v.k: +; X32-NEXT: .long 4 +; X32-NEXT: .long 8 +; X32-NEXT: .long 0 +; X32-NEXT: .long 0 + +; X32-NOT: __emutls_t.k: + +;;;;; 64-bit targets + +; X64: .data +; X64-LABEL: __emutls_v.i: +; X64-NEXT: .quad 4 +; X64-NEXT: .quad 4 +; X64-NEXT: .quad 0 +; X64-NEXT: .quad __emutls_t.i + +; X64: .section .rodata, +; X64-LABEL: __emutls_t.i: +; X64-NEXT: .long 15 + +; X64: .data +; X64-LABEL: __emutls_v.j: +; X64-NEXT: .quad 4 +; X64-NEXT: .quad 4 +; X64-NEXT: .quad 0 +; X64-NEXT: .quad __emutls_t.j + +; X64: .section .rodata, +; X64-LABEL: __emutls_t.j: +; X64-NEXT: .long 42 + +; X64: .data +; X64-LABEL: __emutls_v.k: +; X64-NEXT: .quad 4 +; X64-NEXT: .quad 8 +; X64-NEXT: .quad 0 +; X64-NEXT: .quad 0 + +; X64-NOT: __emutls_t.k: diff --git a/test/CodeGen/X86/emutls-pie.ll b/test/CodeGen/X86/emutls-pie.ll new file mode 100644 index 000000000000..45e5c38c0d8a --- /dev/null +++ b/test/CodeGen/X86/emutls-pie.ll @@ -0,0 +1,131 @@ +; RUN: llc < %s -emulated-tls -march=x86 -mcpu=generic -mtriple=i386-linux-gnu -relocation-model=pic -enable-pie \ +; RUN: | FileCheck -check-prefix=X32 %s +; RUN: llc < %s -emulated-tls -march=x86-64 -mcpu=generic -mtriple=x86_64-linux-gnu -relocation-model=pic -enable-pie \ +; RUN: | FileCheck -check-prefix=X64 %s +; RUN: llc < %s -emulated-tls -march=x86 -mcpu=generic -mtriple=i386-linux-android -relocation-model=pic -enable-pie \ +; RUN: | FileCheck -check-prefix=X32 %s +; RUN: llc < %s -emulated-tls -march=x86-64 -mcpu=generic -mtriple=x86_64-linux-android -relocation-model=pic -enable-pie \ +; RUN: | FileCheck -check-prefix=X64 %s + +; Use my_emutls_get_address like __emutls_get_address. +@my_emutls_v_xyz = external global i8*, align 4 +declare i8* @my_emutls_get_address(i8*) + +define i32 @my_get_xyz() { +; X32-LABEL: my_get_xyz: +; X32: movl my_emutls_v_xyz@GOT(%ebx), %eax +; X32-NEXT: movl %eax, (%esp) +; X32-NEXT: calll my_emutls_get_address@PLT +; X32-NEXT: movl (%eax), %eax +; X32-NEXT: addl $8, %esp +; X32-NEXT: popl %ebx +; X32-NEXT: retl +; X64-LABEL: my_get_xyz: +; X64: movq my_emutls_v_xyz@GOTPCREL(%rip), %rdi +; X64-NEXT: callq my_emutls_get_address@PLT +; X64-NEXT: movl (%rax), %eax +; X64-NEXT: popq %rcx +; X64-NEXT: retq + +entry: + %call = call i8* @my_emutls_get_address(i8* bitcast (i8** @my_emutls_v_xyz to i8*)) + %0 = bitcast i8* %call to i32* + %1 = load i32, i32* %0, align 4 + ret i32 %1 +} + +@i = thread_local global i32 15 +@i2 = external thread_local global i32 + +define i32 @f1() { +; X32-LABEL: f1: +; X32: movl __emutls_v.i@GOT(%ebx), %eax +; X32-NEXT: movl %eax, (%esp) +; X32-NEXT: calll __emutls_get_address@PLT +; X32-NEXT: movl (%eax), %eax +; X32-NEXT: addl $8, %esp +; X32-NEXT: popl %ebx +; X32-NEXT: retl +; X64-LABEL: f1: +; X64: movq __emutls_v.i@GOTPCREL(%rip), %rdi +; X64-NEXT: callq __emutls_get_address@PLT +; X64-NEXT: movl (%rax), %eax +; X64-NEXT: popq %rcx +; X64-NEXT: retq + +entry: + %tmp1 = load i32, i32* @i + ret i32 %tmp1 +} + +define i32* @f2() { +; X32-LABEL: f2: +; X32: movl __emutls_v.i@GOT(%ebx), %eax +; X32-NEXT: movl %eax, (%esp) +; X32-NEXT: calll __emutls_get_address@PLT +; X64-LABEL: f2: +; X64: movq __emutls_v.i@GOTPCREL(%rip), %rdi +; X64-NEXT: callq __emutls_get_address@PLT + +entry: + ret i32* @i +} + +define i32 @f3() { +; X32-LABEL: f3: +; X32: movl __emutls_v.i2@GOT(%ebx), %eax +; X32-NEXT: movl %eax, (%esp) +; X32-NEXT: calll __emutls_get_address@PLT +; X64-LABEL: f3: +; X64: movq __emutls_v.i2@GOTPCREL(%rip), %rdi +; X64-NEXT: callq __emutls_get_address@PLT + +entry: + %tmp1 = load i32, i32* @i2 + ret i32 %tmp1 +} + +define i32* @f4() { +; X32-LABEL: f4: +; X32: movl __emutls_v.i2@GOT(%ebx), %eax +; X32-NEXT: movl %eax, (%esp) +; X32-NEXT: calll __emutls_get_address@PLT +; X64-LABEL: f4: +; X64: movq __emutls_v.i2@GOTPCREL(%rip), %rdi +; X64-NEXT: callq __emutls_get_address@PLT + +entry: + ret i32* @i2 +} + +;;;;; 32-bit targets + +; X32: .data +; X32-LABEL: __emutls_v.i: +; X32-NEXT: .long 4 +; X32-NEXT: .long 4 +; X32-NEXT: .long 0 +; X32-NEXT: .long __emutls_t.i + +; X32: .section .rodata, +; X32-LABEL: __emutls_t.i: +; X32-NEXT: .long 15 + +; X32-NOT: __emutls_v.i2 +; X32-NOT: __emutls_t.i2 + +;;;;; 64-bit targets + +; X64: .data +; X64-LABEL: __emutls_v.i: +; X64-NEXT: .quad 4 +; X64-NEXT: .quad 4 +; X64-NEXT: .quad 0 +; X64-NEXT: .quad __emutls_t.i + +; X64: .section .rodata, +; X64-LABEL: __emutls_t.i: +; X64-NEXT: .long 15 + +; X64-NOT: __emutls_v.i2 +; X64-NOT: __emutls_t.i2 diff --git a/test/CodeGen/X86/emutls.ll b/test/CodeGen/X86/emutls.ll new file mode 100644 index 000000000000..9266fe962df2 --- /dev/null +++ b/test/CodeGen/X86/emutls.ll @@ -0,0 +1,347 @@ +; RUN: llc < %s -emulated-tls -march=x86 -mtriple=i386-linux-gnu | FileCheck -check-prefix=X32 %s +; RUN: llc < %s -emulated-tls -march=x86-64 -mtriple=x86_64-linux-gnu | FileCheck -check-prefix=X64 %s +; RUN: llc < %s -emulated-tls -march=x86 -mtriple=x86-linux-android | FileCheck -check-prefix=X32 %s +; RUN: llc < %s -emulated-tls -march=x86-64 -mtriple=x86_64-linux-android | FileCheck -check-prefix=X64 %s + +; Copied from tls.ll; emulated TLS model is not implemented +; for *-pc-win32 and *-pc-winows targets yet. + +; Use my_emutls_get_address like __emutls_get_address. +@my_emutls_v_xyz = external global i8*, align 4 +declare i8* @my_emutls_get_address(i8*) + +define i32 @my_get_xyz() { +; X32-LABEL: my_get_xyz: +; X32: movl $my_emutls_v_xyz, (%esp) +; X32-NEXT: calll my_emutls_get_address +; X32-NEXT: movl (%eax), %eax +; X32-NEXT: addl $12, %esp +; X32-NEXT: retl +; X64-LABEL: my_get_xyz: +; X64: movl $my_emutls_v_xyz, %edi +; X64-NEXT: callq my_emutls_get_address +; X64-NEXT: movl (%rax), %eax +; X64-NEXT: popq %rcx +; X64-NEXT: retq + +entry: + %call = call i8* @my_emutls_get_address(i8* bitcast (i8** @my_emutls_v_xyz to i8*)) + %0 = bitcast i8* %call to i32* + %1 = load i32, i32* %0, align 4 + ret i32 %1 +} + +@i1 = thread_local global i32 15 +@i2 = external thread_local global i32 +@i3 = internal thread_local global i32 15 +@i4 = hidden thread_local global i32 15 +@i5 = external hidden thread_local global i32 +@s1 = thread_local global i16 15 +@b1 = thread_local global i8 0 + +define i32 @f1() { +; X32-LABEL: f1: +; X32: movl $__emutls_v.i1, (%esp) +; X32-NEXT: calll __emutls_get_address +; X32-NEXT: movl (%eax), %eax +; X32-NEXT: addl $12, %esp +; X32-NEXT: retl +; X64-LABEL: f1: +; X64: movl $__emutls_v.i1, %edi +; X64-NEXT: callq __emutls_get_address +; X64-NEXT: movl (%rax), %eax +; X64-NEXT: popq %rcx +; X64-NEXT: retq + +entry: + %tmp1 = load i32, i32* @i1 + ret i32 %tmp1 +} + +define i32* @f2() { +; X32-LABEL: f2: +; X32: movl $__emutls_v.i1, (%esp) +; X32-NEXT: calll __emutls_get_address +; X32-NEXT: addl $12, %esp +; X32-NEXT: retl +; X64-LABEL: f2: +; X64: movl $__emutls_v.i1, %edi +; X64-NEXT: callq __emutls_get_address +; X64-NEXT: popq %rcx +; X64-NEXT: retq + +entry: + ret i32* @i1 +} + +define i32 @f3() nounwind { +; X32-LABEL: f3: +; X32: movl $__emutls_v.i2, (%esp) +; X32-NEXT: calll __emutls_get_address +; X32-NEXT: movl (%eax), %eax +; X32-NEXT: addl $12, %esp +; X32-NEXT: retl + +entry: + %tmp1 = load i32, i32* @i2 + ret i32 %tmp1 +} + +define i32* @f4() { +; X32-LABEL: f4: +; X32: movl $__emutls_v.i2, (%esp) +; X32-NEXT: calll __emutls_get_address +; X32-NEXT: addl $12, %esp +; X32-NEXT: retl + +entry: + ret i32* @i2 +} + +define i32 @f5() nounwind { +; X32-LABEL: f5: +; X32: movl $__emutls_v.i3, (%esp) +; X32-NEXT: calll __emutls_get_address +; X32-NEXT: movl (%eax), %eax +; X32-NEXT: addl $12, %esp +; X32-NEXT: retl + +entry: + %tmp1 = load i32, i32* @i3 + ret i32 %tmp1 +} + +define i32* @f6() { +; X32-LABEL: f6: +; X32: movl $__emutls_v.i3, (%esp) +; X32-NEXT: calll __emutls_get_address +; X32-NEXT: addl $12, %esp +; X32-NEXT: retl + +entry: + ret i32* @i3 +} + +define i32 @f7() { +; X32-LABEL: f7: +; X32: movl $__emutls_v.i4, (%esp) +; X32-NEXT: calll __emutls_get_address +; X32-NEXT: movl (%eax), %eax +; X32-NEXT: addl $12, %esp +; X32-NEXT: retl + +entry: + %tmp1 = load i32, i32* @i4 + ret i32 %tmp1 +} + +define i32* @f8() { +; X32-LABEL: f8: +; X32: movl $__emutls_v.i4, (%esp) +; X32-NEXT: calll __emutls_get_address +; X32-NEXT: addl $12, %esp +; X32-NEXT: retl + +entry: + ret i32* @i4 +} + +define i32 @f9() { +; X32-LABEL: f9: +; X32: movl $__emutls_v.i5, (%esp) +; X32-NEXT: calll __emutls_get_address +; X32-NEXT: movl (%eax), %eax +; X32-NEXT: addl $12, %esp +; X32-NEXT: retl + +entry: + %tmp1 = load i32, i32* @i5 + ret i32 %tmp1 +} + +define i32* @f10() { +; X32-LABEL: f10: +; X32: movl $__emutls_v.i5, (%esp) +; X32-NEXT: calll __emutls_get_address +; X32-NEXT: addl $12, %esp +; X32-NEXT: retl + +entry: + ret i32* @i5 +} + +define i16 @f11() { +; X32-LABEL: f11: +; X32: movl $__emutls_v.s1, (%esp) +; X32-NEXT: calll __emutls_get_address +; X32-NEXT: movzwl (%eax), %eax +; X32-NEXT: addl $12, %esp +; X32-NEXT: retl + +entry: + %tmp1 = load i16, i16* @s1 + ret i16 %tmp1 +} + +define i32 @f12() { +; X32-LABEL: f12: +; X32: movl $__emutls_v.s1, (%esp) +; X32-NEXT: calll __emutls_get_address +; X32-NEXT: movswl (%eax), %eax +; X32-NEXT: addl $12, %esp +; X32-NEXT: retl + +entry: + %tmp1 = load i16, i16* @s1 + %tmp2 = sext i16 %tmp1 to i32 + ret i32 %tmp2 +} + +define i8 @f13() { +; X32-LABEL: f13: +; X32: movl $__emutls_v.b1, (%esp) +; X32-NEXT: calll __emutls_get_address +; X32-NEXT: movb (%eax), %al +; X32-NEXT: addl $12, %esp +; X32-NEXT: retl + +entry: + %tmp1 = load i8, i8* @b1 + ret i8 %tmp1 +} + +define i32 @f14() { +; X32-LABEL: f14: +; X32: movl $__emutls_v.b1, (%esp) +; X32-NEXT: calll __emutls_get_address +; X32-NEXT: movsbl (%eax), %eax +; X32-NEXT: addl $12, %esp +; X32-NEXT: retl + +entry: + %tmp1 = load i8, i8* @b1 + %tmp2 = sext i8 %tmp1 to i32 + ret i32 %tmp2 +} + +;;;;;;;;;;;;;; 32-bit __emutls_v. and __emutls_t. + +; X32 .section .data.rel.local, +; X32-LABEL: __emutls_v.i1: +; X32-NEXT: .long 4 +; X32-NEXT: .long 4 +; X32-NEXT: .long 0 +; X32-NEXT: .long __emutls_t.i1 + +; X32 .section .rodata, +; X32-LABEL: __emutls_t.i1: +; X32-NEXT: .long 15 + +; X32-NOT: __emutls_v.i2 + +; X32 .section .data.rel.local, +; X32-LABEL: __emutls_v.i3: +; X32-NEXT: .long 4 +; X32-NEXT: .long 4 +; X32-NEXT: .long 0 +; X32-NEXT: .long __emutls_t.i3 + +; X32 .section .rodata, +; X32-LABEL: __emutls_t.i3: +; X32-NEXT: .long 15 + +; X32 .section .data.rel.local, +; X32-LABEL: __emutls_v.i4: +; X32-NEXT: .long 4 +; X32-NEXT: .long 4 +; X32-NEXT: .long 0 +; X32-NEXT: .long __emutls_t.i4 + +; X32 .section .rodata, +; X32-LABEL: __emutls_t.i4: +; X32-NEXT: .long 15 + +; X32-NOT: __emutls_v.i5: +; X32 .hidden __emutls_v.i5 +; X32-NOT: __emutls_v.i5: + +; X32 .section .data.rel.local, +; X32-LABEL: __emutls_v.s1: +; X32-NEXT: .long 2 +; X32-NEXT: .long 2 +; X32-NEXT: .long 0 +; X32-NEXT: .long __emutls_t.s1 + +; X32 .section .rodata, +; X32-LABEL: __emutls_t.s1: +; X32-NEXT: .short 15 + +; X32 .section .data.rel.local, +; X32-LABEL: __emutls_v.b1: +; X32-NEXT: .long 1 +; X32-NEXT: .long 1 +; X32-NEXT: .long 0 +; X32-NEXT: .long 0 + +; X32-NOT: __emutls_t.b1 + +;;;;;;;;;;;;;; 64-bit __emutls_v. and __emutls_t. + +; X64 .section .data.rel.local, +; X64-LABEL: __emutls_v.i1: +; X64-NEXT: .quad 4 +; X64-NEXT: .quad 4 +; X64-NEXT: .quad 0 +; X64-NEXT: .quad __emutls_t.i1 + +; X64 .section .rodata, +; X64-LABEL: __emutls_t.i1: +; X64-NEXT: .long 15 + +; X64-NOT: __emutls_v.i2 + +; X64 .section .data.rel.local, +; X64-LABEL: __emutls_v.i3: +; X64-NEXT: .quad 4 +; X64-NEXT: .quad 4 +; X64-NEXT: .quad 0 +; X64-NEXT: .quad __emutls_t.i3 + +; X64 .section .rodata, +; X64-LABEL: __emutls_t.i3: +; X64-NEXT: .long 15 + +; X64 .section .data.rel.local, +; X64-LABEL: __emutls_v.i4: +; X64-NEXT: .quad 4 +; X64-NEXT: .quad 4 +; X64-NEXT: .quad 0 +; X64-NEXT: .quad __emutls_t.i4 + +; X64 .section .rodata, +; X64-LABEL: __emutls_t.i4: +; X64-NEXT: .long 15 + +; X64-NOT: __emutls_v.i5: +; X64 .hidden __emutls_v.i5 +; X64-NOT: __emutls_v.i5: + +; X64 .section .data.rel.local, +; X64-LABEL: __emutls_v.s1: +; X64-NEXT: .quad 2 +; X64-NEXT: .quad 2 +; X64-NEXT: .quad 0 +; X64-NEXT: .quad __emutls_t.s1 + +; X64 .section .rodata, +; X64-LABEL: __emutls_t.s1: +; X64-NEXT: .short 15 + +; X64 .section .data.rel.local, +; X64-LABEL: __emutls_v.b1: +; X64-NEXT: .quad 1 +; X64-NEXT: .quad 1 +; X64-NEXT: .quad 0 +; X64-NEXT: .quad 0 + +; X64-NOT: __emutls_t.b1 diff --git a/test/CodeGen/X86/emutls_generic.ll b/test/CodeGen/X86/emutls_generic.ll new file mode 100644 index 000000000000..b99a195426c2 --- /dev/null +++ b/test/CodeGen/X86/emutls_generic.ll @@ -0,0 +1,107 @@ +; RUN: llc < %s -emulated-tls -mtriple=i686-linux-android -relocation-model=pic \ +; RUN: | FileCheck -check-prefix=X86_32 %s +; RUN: llc < %s -emulated-tls -mtriple=x86_64-linux-android -march=x86 -relocation-model=pic \ +; RUN: | FileCheck -check-prefix=X86_32 %s +; RUN: llc < %s -emulated-tls -mtriple=x86_64-linux-android -relocation-model=pic \ +; RUN: | FileCheck -check-prefix=X86_64 %s +; RUN: llc < %s -emulated-tls -march=x86 -mtriple=i386-linux-gnu -relocation-model=pic \ +; RUN: | FileCheck %s + +; Make sure that TLS symbols are emitted in expected order. + +@external_x = external thread_local global i32, align 8 +@external_y = thread_local global i8 7, align 2 +@internal_y = internal thread_local global i64 9, align 16 + +define i32* @get_external_x() { +entry: + ret i32* @external_x +} + +define i8* @get_external_y() { +entry: + ret i8* @external_y +} + +define i64* @get_internal_y() { +entry: + ret i64* @internal_y +} + +; CHECK-LABEL: get_external_x: +; CHECK-NOT: _tls_get_address +; CHECK: __emutls_get_address +; CHECK-LABEL: get_external_y: +; CHECK: __emutls_get_address +; CHECK-NOT: _tls_get_address +; CHECK-LABEL: get_internal_y: +; CHECK-NOT: __emutls_t.external_x: +; CHECK-NOT: __emutls_v.external_x: +; CHECK-LABEL: __emutls_v.external_y: +; CHECK-LABEL: __emutls_t.external_y: +; CHECK: __emutls_t.external_y +; CHECK-LABEL: __emutls_v.internal_y: +; CHECK-LABEL: __emutls_t.internal_y: +; CHECK: __emutls_t.internal_y + +; X86_32-LABEL: get_external_x: +; X86_32: movl __emutls_v.external_x +; X86_32: calll __emutls_get_address +; X86_32-LABEL: get_external_y: +; X86_32: movl __emutls_v.external_y +; X86_32: calll __emutls_get_address +; X86_32-LABEL: get_internal_y: +; X86_32: movl __emutls_v.internal_y +; X86_32: calll __emutls_get_address +; X86_32-NOT: __emutls_t.external_x +; X86_32-NOT: __emutls_v.external_x: +; X86_32: .data +; X86_32: .align 4 +; X86_32-LABEL: __emutls_v.external_y: +; X86_32-NEXT: .long 1 +; X86_32-NEXT: .long 2 +; X86_32-NEXT: .long 0 +; X86_32-NEXT: .long __emutls_t.external_y +; X86_32: .section .rodata, +; X86_32-LABEL: __emutls_t.external_y: +; X86_32-NEXT: .byte 7 +; X86_32: .data +; X86_32: .align 4 +; X86_32-LABEL: __emutls_v.internal_y: +; X86_32-NEXT: .long 8 +; X86_32-NEXT: .long 16 +; X86_32-NEXT: .long 0 +; X86_32-NEXT: .long __emutls_t.internal_y +; X86_32-LABEL: __emutls_t.internal_y: +; X86_32-NEXT: .quad 9 +; X86_64-LABEL: get_external_x: +; X86_64: __emutls_v.external_x +; X86_64: __emutls_get_address +; X86_64-LABEL: get_external_y: +; X86_64: __emutls_v.external_y +; X86_64: __emutls_get_address +; X86_64-LABEL: get_internal_y: +; X86_64: __emutls_v.internal_y +; X86_64: __emutls_get_address +; X86_64-NOT: __emutls_t.external_x +; X86_64-NOT: __emutls_v.external_x: +; X86_64: .align 8 +; X86_64-LABEL: __emutls_v.external_y: +; X86_64-NEXT: .quad 1 +; X86_64-NEXT: .quad 2 +; X86_64-NEXT: .quad 0 +; X86_64-NEXT: .quad __emutls_t.external_y +; X86_64-NOT: __emutls_v.external_x: +; X86_64: .section .rodata, +; X86_64-LABEL: __emutls_t.external_y: +; X86_64-NEXT: .byte 7 +; X86_64: .data +; X86_64: .align 8 +; X86_64-LABEL: __emutls_v.internal_y: +; X86_64-NEXT: .quad 8 +; X86_64-NEXT: .quad 16 +; X86_64-NEXT: .quad 0 +; X86_64-NEXT: .quad __emutls_t.internal_y +; X86_64: .section .rodata, +; X86_64-LABEL: __emutls_t.internal_y: +; X86_64-NEXT: .quad 9 diff --git a/test/CodeGen/X86/exedeps-movq.ll b/test/CodeGen/X86/exedeps-movq.ll index a5873be6f27f..c1c60981edf5 100644 --- a/test/CodeGen/X86/exedeps-movq.ll +++ b/test/CodeGen/X86/exedeps-movq.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse2 | FileCheck %s --check-prefix=SSE ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx | FileCheck %s --check-prefix=AVX @@ -66,3 +67,21 @@ define void @store_int(<4 x i32> %x, <2 x float>* %p) { ret void } +define void @store_h_double(<2 x double> %x, i64* %p) { +; SSE-LABEL: store_h_double: +; SSE: # BB#0: +; SSE-NEXT: addpd %xmm0, %xmm0 +; SSE-NEXT: movhpd %xmm0, (%rdi) +; SSE-NEXT: retq +; +; AVX-LABEL: store_h_double: +; AVX: # BB#0: +; AVX-NEXT: vaddpd %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vmovhpd %xmm0, (%rdi) +; AVX-NEXT: retq + %a = fadd <2 x double> %x, %x + %b = extractelement <2 x double> %a, i32 1 + %c = bitcast double %b to i64 + store i64 %c, i64* %p + ret void +} diff --git a/test/CodeGen/X86/expand-vr64-gr64-copy.mir b/test/CodeGen/X86/expand-vr64-gr64-copy.mir new file mode 100644 index 000000000000..8ce1c7eaae70 --- /dev/null +++ b/test/CodeGen/X86/expand-vr64-gr64-copy.mir @@ -0,0 +1,36 @@ +# RUN: llc -run-pass postrapseudos -mtriple=x86_64-unknown-unknown -mattr=+3dnow -o /dev/null %s | FileCheck %s +# This test verifies that the ExpandPostRA pass expands the GR64 <-> VR64 +# copies into appropriate MMX_MOV instructions. + +--- | + + define <2 x i32> @test_pswapdsi(<2 x i32> %a) nounwind readnone { + entry: + %0 = bitcast <2 x i32> %a to x86_mmx + %1 = tail call x86_mmx @llvm.x86.3dnowa.pswapd(x86_mmx %0) + %2 = bitcast x86_mmx %1 to <2 x i32> + ret <2 x i32> %2 + } + + declare x86_mmx @llvm.x86.3dnowa.pswapd(x86_mmx) nounwind readnone + +... +--- +name: test_pswapdsi +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: %xmm0 + + %xmm0 = PSHUFDri killed %xmm0, -24 + MOVPQI2QImr %rsp, 1, _, -8, _, killed %xmm0 + %mm0 = PSWAPDrm %rsp, 1, _, -8, _ + ; CHECK: %rax = MMX_MOVD64from64rr %mm0 + ; CHECK-NEXT: %mm0 = MMX_MOVD64to64rr %rax + %rax = COPY %mm0 + %mm0 = COPY %rax + MMX_MOVQ64mr %rsp, 1, _, -16, _, killed %mm0 + %xmm0 = MOVQI2PQIrm %rsp, 1, _, -16, _ + %xmm0 = PSHUFDri killed %xmm0, -44 + RETQ %xmm0 +... diff --git a/test/CodeGen/X86/extractelement-legalization-cycle.ll b/test/CodeGen/X86/extractelement-legalization-cycle.ll new file mode 100644 index 000000000000..d75f03ba1680 --- /dev/null +++ b/test/CodeGen/X86/extractelement-legalization-cycle.ll @@ -0,0 +1,21 @@ +; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s + +; When the extractelement is converted to a load the store can be re-used. +; This will, however, introduce a cycle into the selection DAG (the load +; of the extractelement index is dependent on the store, and so after the +; conversion it becomes dependent on the new load, which is dependent on +; the index). Make sure we skip the store, and conservatively instead +; use a store to the stack. + +define float @foo(i32* %i, <4 x float>* %v) { +; CHECK-LABEL: foo: +; CHECK: movaps %xmm0, -[[OFFSET:[0-9]+]](%rsp) +; CHECK: movss -[[OFFSET]](%rsp,{{.*}}), %xmm0 {{.*}} +; CHECK-NEXT: retq + %1 = load <4 x float>, <4 x float>* %v, align 16 + %mul = fmul <4 x float> %1, %1 + store <4 x float> %mul, <4 x float>* %v, align 16 + %2 = load i32, i32* %i, align 4 + %vecext = extractelement <4 x float> %mul, i32 %2 + ret float %vecext +} diff --git a/test/CodeGen/X86/extractelement-shuffle.ll b/test/CodeGen/X86/extractelement-shuffle.ll index d1ba9a845800..1b04c41d5c6f 100644 --- a/test/CodeGen/X86/extractelement-shuffle.ll +++ b/test/CodeGen/X86/extractelement-shuffle.ll @@ -1,4 +1,5 @@ ; RUN: llc < %s +; REQUIRES: default_triple ; Examples that exhibits a bug in DAGCombine. The case is triggered by the ; following program. The bug is DAGCombine assumes that the bit convert diff --git a/test/CodeGen/X86/fadd-combines.ll b/test/CodeGen/X86/fadd-combines.ll new file mode 100644 index 000000000000..2df0e06dc252 --- /dev/null +++ b/test/CodeGen/X86/fadd-combines.ll @@ -0,0 +1,224 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=x86_64-unknown-unknown < %s | FileCheck %s + +define float @fadd_zero_f32(float %x) #0 { +; CHECK-LABEL: fadd_zero_f32: +; CHECK: # BB#0: +; CHECK-NEXT: retq + %y = fadd float %x, 0.0 + ret float %y +} + +define <4 x float> @fadd_zero_4f32(<4 x float> %x) #0 { +; CHECK-LABEL: fadd_zero_4f32: +; CHECK: # BB#0: +; CHECK-NEXT: retq + %y = fadd <4 x float> %x, zeroinitializer + ret <4 x float> %y +} + +; CHECK: float 3 +define float @fadd_2const_f32(float %x) #0 { +; CHECK-LABEL: fadd_2const_f32: +; CHECK: # BB#0: +; CHECK-NEXT: addss {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq + %y = fadd float %x, 1.0 + %z = fadd float %y, 2.0 + ret float %z +} + +; CHECK: float 5 +; CHECK: float 5 +; CHECK: float 5 +; CHECK: float 5 +define <4 x float> @fadd_2const_4f32(<4 x float> %x) #0 { +; CHECK-LABEL: fadd_2const_4f32: +; CHECK: # BB#0: +; CHECK-NEXT: addps {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq + %y = fadd <4 x float> %x, + %z = fadd <4 x float> %y, + ret <4 x float> %z +} + +; CHECK: float 3 +define float @fadd_x_fmul_x_c_f32(float %x) #0 { +; CHECK-LABEL: fadd_x_fmul_x_c_f32: +; CHECK: # BB#0: +; CHECK-NEXT: mulss {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq + %y = fmul float %x, 2.0 + %z = fadd float %x, %y + ret float %z +} + +; CHECK: float 2 +; CHECK: float 3 +; CHECK: float 4 +; CHECK: float 5 +define <4 x float> @fadd_x_fmul_x_c_4f32(<4 x float> %x) #0 { +; CHECK-LABEL: fadd_x_fmul_x_c_4f32: +; CHECK: # BB#0: +; CHECK-NEXT: mulps {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq + %y = fmul <4 x float> %x, + %z = fadd <4 x float> %x, %y + ret <4 x float> %z +} + +; CHECK: float 3 +define float @fadd_fmul_x_c_x_f32(float %x) #0 { +; CHECK-LABEL: fadd_fmul_x_c_x_f32: +; CHECK: # BB#0: +; CHECK-NEXT: mulss {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq + %y = fmul float %x, 2.0 + %z = fadd float %y, %x + ret float %z +} + +; CHECK: float 2 +; CHECK: float 3 +; CHECK: float 4 +; CHECK: float 5 +define <4 x float> @fadd_fmul_x_c_x_4f32(<4 x float> %x) #0 { +; CHECK-LABEL: fadd_fmul_x_c_x_4f32: +; CHECK: # BB#0: +; CHECK-NEXT: mulps {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq + %y = fmul <4 x float> %x, + %z = fadd <4 x float> %y, %x + ret <4 x float> %z +} + +; CHECK: float 4 +define float @fadd_fadd_x_x_fmul_x_c_f32(float %x) #0 { +; CHECK-LABEL: fadd_fadd_x_x_fmul_x_c_f32: +; CHECK: # BB#0: +; CHECK-NEXT: mulss {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq + %y = fadd float %x, %x + %z = fmul float %x, 2.0 + %w = fadd float %y, %z + ret float %w +} + +; CHECK: float 3 +; CHECK: float 4 +; CHECK: float 5 +; CHECK: float 6 +define <4 x float> @fadd_fadd_x_x_fmul_x_c_4f32(<4 x float> %x) #0 { +; CHECK-LABEL: fadd_fadd_x_x_fmul_x_c_4f32: +; CHECK: # BB#0: +; CHECK-NEXT: mulps {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq + %y = fadd <4 x float> %x, %x + %z = fmul <4 x float> %x, + %w = fadd <4 x float> %y, %z + ret <4 x float> %w +} + +; CHECK: float 4 +define float @fadd_fmul_x_c_fadd_x_x_f32(float %x) #0 { +; CHECK-LABEL: fadd_fmul_x_c_fadd_x_x_f32: +; CHECK: # BB#0: +; CHECK-NEXT: mulss {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq + %y = fadd float %x, %x + %z = fmul float %x, 2.0 + %w = fadd float %z, %y + ret float %w +} + +; CHECK: float 3 +; CHECK: float 4 +; CHECK: float 5 +; CHECK: float 6 +define <4 x float> @fadd_fmul_x_c_fadd_x_x_4f32(<4 x float> %x) #0 { +; CHECK-LABEL: fadd_fmul_x_c_fadd_x_x_4f32: +; CHECK: # BB#0: +; CHECK-NEXT: mulps {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq + %y = fadd <4 x float> %x, %x + %z = fmul <4 x float> %x, + %w = fadd <4 x float> %z, %y + ret <4 x float> %w +} + +; CHECK: float 3 +define float @fadd_x_fadd_x_x_f32(float %x) #0 { +; CHECK-LABEL: fadd_x_fadd_x_x_f32: +; CHECK: # BB#0: +; CHECK-NEXT: mulss {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq + %y = fadd float %x, %x + %z = fadd float %x, %y + ret float %z +} + +; CHECK: float 3 +; CHECK: float 3 +; CHECK: float 3 +; CHECK: float 3 +define <4 x float> @fadd_x_fadd_x_x_4f32(<4 x float> %x) #0 { +; CHECK-LABEL: fadd_x_fadd_x_x_4f32: +; CHECK: # BB#0: +; CHECK-NEXT: mulps {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq + %y = fadd <4 x float> %x, %x + %z = fadd <4 x float> %x, %y + ret <4 x float> %z +} + +; CHECK: float 3 +define float @fadd_fadd_x_x_x_f32(float %x) #0 { +; CHECK-LABEL: fadd_fadd_x_x_x_f32: +; CHECK: # BB#0: +; CHECK-NEXT: mulss {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq + %y = fadd float %x, %x + %z = fadd float %y, %x + ret float %z +} + +; CHECK: float 3 +; CHECK: float 3 +; CHECK: float 3 +; CHECK: float 3 +define <4 x float> @fadd_fadd_x_x_x_4f32(<4 x float> %x) #0 { +; CHECK-LABEL: fadd_fadd_x_x_x_4f32: +; CHECK: # BB#0: +; CHECK-NEXT: mulps {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq + %y = fadd <4 x float> %x, %x + %z = fadd <4 x float> %y, %x + ret <4 x float> %z +} + +; CHECK: float 4 +define float @fadd_fadd_x_x_fadd_x_x_f32(float %x) #0 { +; CHECK-LABEL: fadd_fadd_x_x_fadd_x_x_f32: +; CHECK: # BB#0: +; CHECK-NEXT: mulss {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq + %y = fadd float %x, %x + %z = fadd float %y, %y + ret float %z +} + +; CHECK: float 4 +; CHECK: float 4 +; CHECK: float 4 +; CHECK: float 4 +define <4 x float> @fadd_fadd_x_x_fadd_x_x_4f32(<4 x float> %x) #0 { +; CHECK-LABEL: fadd_fadd_x_x_fadd_x_x_4f32: +; CHECK: # BB#0: +; CHECK-NEXT: mulps {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq + %y = fadd <4 x float> %x, %x + %z = fadd <4 x float> %y, %y + ret <4 x float> %z +} + +attributes #0 = { "less-precise-fpmad"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "unsafe-fp-math"="true" } diff --git a/test/CodeGen/X86/fast-isel-bitcasts-avx.ll b/test/CodeGen/X86/fast-isel-bitcasts-avx.ll new file mode 100644 index 000000000000..03cefbc86822 --- /dev/null +++ b/test/CodeGen/X86/fast-isel-bitcasts-avx.ll @@ -0,0 +1,244 @@ +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -fast-isel -fast-isel-abort=1 -asm-verbose=0 | FileCheck %s +; +; Bitcasts between 256-bit vector types are no-ops since no instruction is +; needed for the conversion. + +define <4 x i64> @v8i32_to_v4i64(<8 x i32> %a) { +;CHECK-LABEL: v8i32_to_v4i64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <8 x i32> %a to <4 x i64> + ret <4 x i64> %1 +} + +define <4 x i64> @v16i16_to_v4i64(<16 x i16> %a) { +;CHECK-LABEL: v16i16_to_v4i64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <16 x i16> %a to <4 x i64> + ret <4 x i64> %1 +} + +define <4 x i64> @v32i8_to_v4i64(<32 x i8> %a) { +;CHECK-LABEL: v32i8_to_v4i64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <32 x i8> %a to <4 x i64> + ret <4 x i64> %1 +} + +define <4 x i64> @v4f64_to_v4i64(<4 x double> %a) { +;CHECK-LABEL: v4f64_to_v4i64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x double> %a to <4 x i64> + ret <4 x i64> %1 +} + +define <4 x i64> @v8f32_to_v4i64(<8 x float> %a) { +;CHECK-LABEL: v8f32_to_v4i64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <8 x float> %a to <4 x i64> + ret <4 x i64> %1 +} + +define <8 x i32> @v4i64_to_v8i32(<4 x i64> %a) { +;CHECK-LABEL: v4i64_to_v8i32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x i64> %a to <8 x i32> + ret <8 x i32> %1 +} + +define <8 x i32> @v16i16_to_v8i32(<16 x i16> %a) { +;CHECK-LABEL: v16i16_to_v8i32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <16 x i16> %a to <8 x i32> + ret <8 x i32> %1 +} + +define <8 x i32> @v32i8_to_v8i32(<32 x i8> %a) { +;CHECK-LABEL: v32i8_to_v8i32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <32 x i8> %a to <8 x i32> + ret <8 x i32> %1 +} + +define <8 x i32> @v4f64_to_v8i32(<4 x double> %a) { +;CHECK-LABEL: v4f64_to_v8i32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x double> %a to <8 x i32> + ret <8 x i32> %1 +} + +define <8 x i32> @v8f32_to_v8i32(<8 x float> %a) { +;CHECK-LABEL: v8f32_to_v8i32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <8 x float> %a to <8 x i32> + ret <8 x i32> %1 +} + +define <16 x i16> @v4i64_to_v16i16(<4 x i64> %a) { +;CHECK-LABEL: v4i64_to_v16i16: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x i64> %a to <16 x i16> + ret <16 x i16> %1 +} + +define <16 x i16> @v8i32_to_v16i16(<8 x i32> %a) { +;CHECK-LABEL: v8i32_to_v16i16: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <8 x i32> %a to <16 x i16> + ret <16 x i16> %1 +} + +define <16 x i16> @v32i8_to_v16i16(<32 x i8> %a) { +;CHECK-LABEL: v32i8_to_v16i16: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <32 x i8> %a to <16 x i16> + ret <16 x i16> %1 +} + +define <16 x i16> @v4f64_to_v16i16(<4 x double> %a) { +;CHECK-LABEL: v4f64_to_v16i16: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x double> %a to <16 x i16> + ret <16 x i16> %1 +} + +define <16 x i16> @v8f32_to_v16i16(<8 x float> %a) { +;CHECK-LABEL: v8f32_to_v16i16: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <8 x float> %a to <16 x i16> + ret <16 x i16> %1 +} + +define <32 x i8> @v16i16_to_v32i8(<16 x i16> %a) { +;CHECK-LABEL: v16i16_to_v32i8: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <16 x i16> %a to <32 x i8> + ret <32 x i8> %1 +} + +define <32 x i8> @v4i64_to_v32i8(<4 x i64> %a) { +;CHECK-LABEL: v4i64_to_v32i8: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x i64> %a to <32 x i8> + ret <32 x i8> %1 +} + +define <32 x i8> @v8i32_to_v32i8(<8 x i32> %a) { +;CHECK-LABEL: v8i32_to_v32i8: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <8 x i32> %a to <32 x i8> + ret <32 x i8> %1 +} + +define <32 x i8> @v4f64_to_v32i8(<4 x double> %a) { +;CHECK-LABEL: v4f64_to_v32i8: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x double> %a to <32 x i8> + ret <32 x i8> %1 +} + +define <32 x i8> @v8f32_to_v32i8(<8 x float> %a) { +;CHECK-LABEL: v8f32_to_v32i8: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <8 x float> %a to <32 x i8> + ret <32 x i8> %1 +} + +define <8 x float> @v32i8_to_v8f32(<32 x i8> %a) { +;CHECK-LABEL: v32i8_to_v8f32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <32 x i8> %a to <8 x float> + ret <8 x float> %1 +} + +define <8 x float> @v16i16_to_v8f32(<16 x i16> %a) { +;CHECK-LABEL: v16i16_to_v8f32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <16 x i16> %a to <8 x float> + ret <8 x float> %1 +} + +define <8 x float> @v4i64_to_v8f32(<4 x i64> %a) { +;CHECK-LABEL: v4i64_to_v8f32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x i64> %a to <8 x float> + ret <8 x float> %1 +} + +define <8 x float> @v8i32_to_v8f32(<8 x i32> %a) { +;CHECK-LABEL: v8i32_to_v8f32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <8 x i32> %a to <8 x float> + ret <8 x float> %1 +} + +define <8 x float> @v4f64_to_v8f32(<4 x double> %a) { +;CHECK-LABEL: v4f64_to_v8f32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x double> %a to <8 x float> + ret <8 x float> %1 +} + +define <4 x double> @v8f32_to_v4f64(<8 x float> %a) { +;CHECK-LABEL: v8f32_to_v4f64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <8 x float> %a to <4 x double> + ret <4 x double> %1 +} + +define <4 x double> @v32i8_to_v4f64(<32 x i8> %a) { +;CHECK-LABEL: v32i8_to_v4f64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <32 x i8> %a to <4 x double> + ret <4 x double> %1 +} + +define <4 x double> @v16i16_to_v4f64(<16 x i16> %a) { +;CHECK-LABEL: v16i16_to_v4f64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <16 x i16> %a to <4 x double> + ret <4 x double> %1 +} + +define <4 x double> @v4i64_to_v4f64(<4 x i64> %a) { +;CHECK-LABEL: v4i64_to_v4f64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x i64> %a to <4 x double> + ret <4 x double> %1 +} + +define <4 x double> @v8i32_to_v4f64(<8 x i32> %a) { +;CHECK-LABEL: v8i32_to_v4f64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <8 x i32> %a to <4 x double> + ret <4 x double> %1 +} diff --git a/test/CodeGen/X86/fast-isel-bitcasts.ll b/test/CodeGen/X86/fast-isel-bitcasts.ll new file mode 100644 index 000000000000..892b517fe873 --- /dev/null +++ b/test/CodeGen/X86/fast-isel-bitcasts.ll @@ -0,0 +1,245 @@ +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 -fast-isel -fast-isel-abort=1 -asm-verbose=0 | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -fast-isel -fast-isel-abort=1 -asm-verbose=0 | FileCheck %s +; +; Bitcasts between 128-bit vector types are no-ops since no instruction is +; needed for the conversion. + +define <2 x i64> @v4i32_to_v2i64(<4 x i32> %a) { +;CHECK-LABEL: v4i32_to_v2i64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x i32> %a to <2 x i64> + ret <2 x i64> %1 +} + +define <2 x i64> @v8i16_to_v2i64(<8 x i16> %a) { +;CHECK-LABEL: v8i16_to_v2i64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <8 x i16> %a to <2 x i64> + ret <2 x i64> %1 +} + +define <2 x i64> @v16i8_to_v2i64(<16 x i8> %a) { +;CHECK-LABEL: v16i8_to_v2i64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <16 x i8> %a to <2 x i64> + ret <2 x i64> %1 +} + +define <2 x i64> @v2f64_to_v2i64(<2 x double> %a) { +;CHECK-LABEL: v2f64_to_v2i64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <2 x double> %a to <2 x i64> + ret <2 x i64> %1 +} + +define <2 x i64> @v4f32_to_v2i64(<4 x float> %a) { +;CHECK-LABEL: v4f32_to_v2i64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x float> %a to <2 x i64> + ret <2 x i64> %1 +} + +define <4 x i32> @v2i64_to_v4i32(<2 x i64> %a) { +;CHECK-LABEL: v2i64_to_v4i32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <2 x i64> %a to <4 x i32> + ret <4 x i32> %1 +} + +define <4 x i32> @v8i16_to_v4i32(<8 x i16> %a) { +;CHECK-LABEL: v8i16_to_v4i32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <8 x i16> %a to <4 x i32> + ret <4 x i32> %1 +} + +define <4 x i32> @v16i8_to_v4i32(<16 x i8> %a) { +;CHECK-LABEL: v16i8_to_v4i32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <16 x i8> %a to <4 x i32> + ret <4 x i32> %1 +} + +define <4 x i32> @v2f64_to_v4i32(<2 x double> %a) { +;CHECK-LABEL: v2f64_to_v4i32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <2 x double> %a to <4 x i32> + ret <4 x i32> %1 +} + +define <4 x i32> @v4f32_to_v4i32(<4 x float> %a) { +;CHECK-LABEL: v4f32_to_v4i32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x float> %a to <4 x i32> + ret <4 x i32> %1 +} + +define <8 x i16> @v2i64_to_v8i16(<2 x i64> %a) { +;CHECK-LABEL: v2i64_to_v8i16: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <2 x i64> %a to <8 x i16> + ret <8 x i16> %1 +} + +define <8 x i16> @v4i32_to_v8i16(<4 x i32> %a) { +;CHECK-LABEL: v4i32_to_v8i16: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x i32> %a to <8 x i16> + ret <8 x i16> %1 +} + +define <8 x i16> @v16i8_to_v8i16(<16 x i8> %a) { +;CHECK-LABEL: v16i8_to_v8i16: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <16 x i8> %a to <8 x i16> + ret <8 x i16> %1 +} + +define <8 x i16> @v2f64_to_v8i16(<2 x double> %a) { +;CHECK-LABEL: v2f64_to_v8i16: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <2 x double> %a to <8 x i16> + ret <8 x i16> %1 +} + +define <8 x i16> @v4f32_to_v8i16(<4 x float> %a) { +;CHECK-LABEL: v4f32_to_v8i16: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x float> %a to <8 x i16> + ret <8 x i16> %1 +} + +define <16 x i8> @v8i16_to_v16i8(<8 x i16> %a) { +;CHECK-LABEL: v8i16_to_v16i8: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <8 x i16> %a to <16 x i8> + ret <16 x i8> %1 +} + +define <16 x i8> @v2i64_to_v16i8(<2 x i64> %a) { +;CHECK-LABEL: v2i64_to_v16i8: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <2 x i64> %a to <16 x i8> + ret <16 x i8> %1 +} + +define <16 x i8> @v4i32_to_v16i8(<4 x i32> %a) { +;CHECK-LABEL: v4i32_to_v16i8: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x i32> %a to <16 x i8> + ret <16 x i8> %1 +} + +define <16 x i8> @v2f64_to_v16i8(<2 x double> %a) { +;CHECK-LABEL: v2f64_to_v16i8: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <2 x double> %a to <16 x i8> + ret <16 x i8> %1 +} + +define <16 x i8> @v4f32_to_v16i8(<4 x float> %a) { +;CHECK-LABEL: v4f32_to_v16i8: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x float> %a to <16 x i8> + ret <16 x i8> %1 +} + +define <4 x float> @v16i8_to_v4f32(<16 x i8> %a) { +;CHECK-LABEL: v16i8_to_v4f32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <16 x i8> %a to <4 x float> + ret <4 x float> %1 +} + +define <4 x float> @v8i16_to_v4f32(<8 x i16> %a) { +;CHECK-LABEL: v8i16_to_v4f32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <8 x i16> %a to <4 x float> + ret <4 x float> %1 +} + +define <4 x float> @v2i64_to_v4f32(<2 x i64> %a) { +;CHECK-LABEL: v2i64_to_v4f32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <2 x i64> %a to <4 x float> + ret <4 x float> %1 +} + +define <4 x float> @v4i32_to_v4f32(<4 x i32> %a) { +;CHECK-LABEL: v4i32_to_v4f32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x i32> %a to <4 x float> + ret <4 x float> %1 +} + +define <4 x float> @v2f64_to_v4f32(<2 x double> %a) { +;CHECK-LABEL: v2f64_to_v4f32: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <2 x double> %a to <4 x float> + ret <4 x float> %1 +} + +define <2 x double> @v4f32_to_v2f64(<4 x float> %a) { +;CHECK-LABEL: v4f32_to_v2f64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x float> %a to <2 x double> + ret <2 x double> %1 +} + +define <2 x double> @v16i8_to_v2f64(<16 x i8> %a) { +;CHECK-LABEL: v16i8_to_v2f64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <16 x i8> %a to <2 x double> + ret <2 x double> %1 +} + +define <2 x double> @v8i16_to_v2f64(<8 x i16> %a) { +;CHECK-LABEL: v8i16_to_v2f64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <8 x i16> %a to <2 x double> + ret <2 x double> %1 +} + +define <2 x double> @v2i64_to_v2f64(<2 x i64> %a) { +;CHECK-LABEL: v2i64_to_v2f64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <2 x i64> %a to <2 x double> + ret <2 x double> %1 +} + +define <2 x double> @v4i32_to_v2f64(<4 x i32> %a) { +;CHECK-LABEL: v4i32_to_v2f64: +;CHECK-NEXT: .cfi_startproc +;CHECK-NEXT: ret + %1 = bitcast <4 x i32> %a to <2 x double> + ret <2 x double> %1 +} diff --git a/test/CodeGen/X86/fast-isel-cmp-branch.ll b/test/CodeGen/X86/fast-isel-cmp-branch.ll index d7b64ed3a5b8..e262448468eb 100644 --- a/test/CodeGen/X86/fast-isel-cmp-branch.ll +++ b/test/CodeGen/X86/fast-isel-cmp-branch.ll @@ -1,5 +1,18 @@ -; RUN: llc -O0 -mtriple=x86_64-linux -asm-verbose=false < %s | FileCheck %s -; RUN: llc -O0 -mtriple=x86_64-windows-itanium -asm-verbose=false < %s | FileCheck %s +; RUN: llc -O0 -mtriple=x86_64-linux -asm-verbose=false -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -O0 -mtriple=x86_64-windows-itanium -asm-verbose=false -verify-machineinstrs < %s | FileCheck %s + +; Fast-isel mustn't add a block to the MBB successor/predecessor list twice. +; The machine verifier will catch and complain about this case. +; CHECK-LABEL: baz +; CHECK: retq +define void @baz() { +entry: + br i1 undef, label %exit, label %exit + +exit: + ret void +} + ; rdar://8337108 ; Fast-isel shouldn't try to look through the compare because it's in a diff --git a/test/CodeGen/X86/fast-isel-deadcode.ll b/test/CodeGen/X86/fast-isel-deadcode.ll new file mode 100644 index 000000000000..0a53d60f8352 --- /dev/null +++ b/test/CodeGen/X86/fast-isel-deadcode.ll @@ -0,0 +1,147 @@ +; RUN: llc < %s | FileCheck %s +; +; Generated with clang -O2 -S -emit-llvm +; +; /* Test 1 */ +; extern "C" bool bar (long double); +; __attribute__((optnone)) +; extern "C" bool foo(long double x, long double y) +; { +; return (x == y) || (bar(x)); +; } +; +; /* Test 2 */ +; struct FVector { +; float x, y, z; +; inline __attribute__((always_inline)) FVector(float f): x(f), y(f), z(f) {} +; inline __attribute__((always_inline)) FVector func(float p) const +; { +; if( x == 1.f ) { +; return *this; +; } else if( x < p ) { +; return FVector(0.f); +; } +; return FVector(x); +; } +; }; +; +; __attribute__((optnone)) +; int main() +; { +; FVector v(1.0); +; v = v.func(1.e-8); +; return 0; +; } +; +; ModuleID = 'test.cpp' +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +%struct.FVector = type { float, float, float } + +define zeroext i1 @foo(x86_fp80 %x, x86_fp80 %y) noinline optnone { +entry: + %x.addr = alloca x86_fp80, align 16 + %y.addr = alloca x86_fp80, align 16 + store x86_fp80 %x, x86_fp80* %x.addr, align 16 + store x86_fp80 %y, x86_fp80* %y.addr, align 16 + %0 = load x86_fp80, x86_fp80* %x.addr, align 16 + %1 = load x86_fp80, x86_fp80* %y.addr, align 16 + %cmp = fcmp oeq x86_fp80 %0, %1 + +; Test 1 +; Make sure that there is no dead code generated +; from Fast-ISel Phi-node handling. We should only +; see one movb of the constant 1, feeding the PHI +; node in lor.end. This covers the code path with +; handlePHINodesInSuccessorBlocks() returning true. +; +; CHECK-LABEL: foo: +; CHECK: movb $1, +; CHECK-NOT: movb $1, +; CHECK-LABEL: .LBB0_1: + + br i1 %cmp, label %lor.end, label %lor.rhs + +lor.rhs: ; preds = %entry + %2 = load x86_fp80, x86_fp80* %x.addr, align 16 + %call = call zeroext i1 @bar(x86_fp80 %2) + br label %lor.end + +lor.end: ; preds = %lor.rhs, %entry + %3 = phi i1 [ true, %entry ], [ %call, %lor.rhs ] + ret i1 %3 +} + +declare zeroext i1 @bar(x86_fp80) + +define i32 @main() noinline optnone { +entry: + %retval = alloca i32, align 4 + %v = alloca %struct.FVector, align 4 + %ref.tmp = alloca %struct.FVector, align 4 + %tmp = alloca { <2 x float>, float }, align 8 + store i32 0, i32* %retval, align 4 + %0 = bitcast %struct.FVector* %v to i8* + call void @llvm.lifetime.start(i64 12, i8* %0) nounwind + %x.i = getelementptr inbounds %struct.FVector, %struct.FVector* %v, i64 0, i32 0 + store float 1.000000e+00, float* %x.i, align 4 + %y.i = getelementptr inbounds %struct.FVector, %struct.FVector* %v, i64 0, i32 1 + store float 1.000000e+00, float* %y.i, align 4 + %z.i = getelementptr inbounds %struct.FVector, %struct.FVector* %v, i64 0, i32 2 + store float 1.000000e+00, float* %z.i, align 4 + %x.i.1 = getelementptr inbounds %struct.FVector, %struct.FVector* %v, i64 0, i32 0 + %1 = load float, float* %x.i.1, align 4 + %cmp.i = fcmp oeq float %1, 1.000000e+00 + br i1 %cmp.i, label %if.then.i, label %if.else.i + +if.then.i: ; preds = %entry + %retval.sroa.0.0..sroa_cast.i = bitcast %struct.FVector* %v to <2 x float>* + %retval.sroa.0.0.copyload.i = load <2 x float>, <2 x float>* %retval.sroa.0.0..sroa_cast.i, align 4 + %retval.sroa.6.0..sroa_idx16.i = getelementptr inbounds %struct.FVector, %struct.FVector* %v, i64 0, i32 2 + %retval.sroa.6.0.copyload.i = load float, float* %retval.sroa.6.0..sroa_idx16.i, align 4 + br label %func.exit + +if.else.i: ; preds = %entry + +; Test 2 +; In order to feed the first PHI node in func.exit handlePHINodesInSuccessorBlocks() +; generates a local value instruction, but it cannot handle the second PHI node and +; returns false to let SelectionDAGISel handle both cases. Make sure the generated +; local value instruction is removed. +; CHECK-LABEL: main: +; CHECK-LABEL: .LBB1_2: +; CHECK: xorps [[REG:%xmm[0-7]]], [[REG]] +; CHECK-NOT: xorps [[REG]], [[REG]] +; CHECK-LABEL: .LBB1_3: + + %cmp3.i = fcmp olt float %1, 0x3E45798EE0000000 + br i1 %cmp3.i, label %func.exit, label %if.end.5.i + +if.end.5.i: ; preds = %if.else.i + %retval.sroa.0.0.vec.insert13.i = insertelement <2 x float> undef, float %1, i32 0 + %retval.sroa.0.4.vec.insert15.i = insertelement <2 x float> %retval.sroa.0.0.vec.insert13.i, float %1, i32 1 + br label %func.exit + +func.exit: ; preds = %if.then.i, %if.else.i, %if.end.5.i + %retval.sroa.6.0.i = phi float [ %retval.sroa.6.0.copyload.i, %if.then.i ], [ %1, %if.end.5.i ], [ 0.000000e+00, %if.else.i ] + %retval.sroa.0.0.i = phi <2 x float> [ %retval.sroa.0.0.copyload.i, %if.then.i ], [ %retval.sroa.0.4.vec.insert15.i, %if.end.5.i ], [ zeroinitializer, %if.else.i ] + %.fca.0.insert.i = insertvalue { <2 x float>, float } undef, <2 x float> %retval.sroa.0.0.i, 0 + %.fca.1.insert.i = insertvalue { <2 x float>, float } %.fca.0.insert.i, float %retval.sroa.6.0.i, 1 + store { <2 x float>, float } %.fca.1.insert.i, { <2 x float>, float }* %tmp, align 8 + %2 = bitcast { <2 x float>, float }* %tmp to i8* + %3 = bitcast %struct.FVector* %ref.tmp to i8* + call void @llvm.memcpy.p0i8.p0i8.i64(i8* %3, i8* %2, i64 12, i32 4, i1 false) + %4 = bitcast %struct.FVector* %v to i8* + %5 = bitcast %struct.FVector* %ref.tmp to i8* + call void @llvm.memcpy.p0i8.p0i8.i64(i8* %4, i8* %5, i64 12, i32 4, i1 false) + %6 = bitcast %struct.FVector* %v to i8* + call void @llvm.lifetime.end(i64 12, i8* %6) nounwind + ret i32 0 +} + +declare void @llvm.lifetime.start(i64, i8* nocapture) argmemonly nounwind + +declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i32, i1) argmemonly nounwind + +declare void @llvm.lifetime.end(i64, i8* nocapture) argmemonly nounwind diff --git a/test/CodeGen/X86/fast-isel-emutls.ll b/test/CodeGen/X86/fast-isel-emutls.ll new file mode 100644 index 000000000000..cb8012c0fa39 --- /dev/null +++ b/test/CodeGen/X86/fast-isel-emutls.ll @@ -0,0 +1,48 @@ +; RUN: llc < %s -emulated-tls -march=x86 -relocation-model=pic -mtriple=i686-unknown-linux-gnu -fast-isel | FileCheck %s +; PR3654 + +@v = thread_local global i32 0 +define i32 @f() nounwind { +entry: + %t = load i32, i32* @v + %s = add i32 %t, 1 + ret i32 %s +} + +; CHECK-LABEL: f: +; CHECK: movl __emutls_v.v@GOT(%ebx), %eax +; CHECK-NEXT: movl %eax, (%esp) +; CHECK-NEXT: calll __emutls_get_address@PLT +; CHECK-NEXT: movl (%eax), %eax + +@alias = internal alias i32, i32* @v +define i32 @f_alias() nounwind { +entry: + %t = load i32, i32* @v + %s = add i32 %t, 1 + ret i32 %s +} + +; CHECK-LABEL: f_alias: +; CHECK: movl __emutls_v.v@GOT(%ebx), %eax +; CHECK-NEXT: movl %eax, (%esp) +; CHECK-NEXT: calll __emutls_get_address@PLT +; CHECK-NEXT: movl (%eax), %eax + +; Use my_emutls_get_address like __emutls_get_address. +@my_emutls_v_xyz = external global i8*, align 4 +declare i8* @my_emutls_get_address(i8*) + +define i32 @my_get_xyz() { +entry: + %call = call i8* @my_emutls_get_address(i8* bitcast (i8** @my_emutls_v_xyz to i8*)) + %0 = bitcast i8* %call to i32* + %1 = load i32, i32* %0, align 4 + ret i32 %1 +} + +; CHECK-LABEL: my_get_xyz: +; CHECK: movl my_emutls_v_xyz@GOT(%ebx), %eax +; CHECK-NEXT: movl %eax, (%esp) +; CHECK-NEXT: calll my_emutls_get_address@PLT +; CHECK-NEXT: movl (%eax), %eax diff --git a/test/CodeGen/X86/fast-isel-nontemporal.ll b/test/CodeGen/X86/fast-isel-nontemporal.ll new file mode 100644 index 000000000000..6a174dbf5a8a --- /dev/null +++ b/test/CodeGen/X86/fast-isel-nontemporal.ll @@ -0,0 +1,111 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+sse2 -fast-isel -O0 < %s | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+sse4a -fast-isel -O0 < %s | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE4A +; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+avx -fast-isel -O0 < %s | FileCheck %s --check-prefix=ALL --check-prefix=AVX + +define void @test_nti32(i32* nocapture %ptr, i32 %X) { +; ALL-LABEL: test_nti32: +; ALL: # BB#0: # %entry +; ALL-NEXT: movntil %esi, (%rdi) +; ALL-NEXT: retq +entry: + store i32 %X, i32* %ptr, align 4, !nontemporal !1 + ret void +} + +define void @test_nti64(i64* nocapture %ptr, i64 %X) { +; ALL-LABEL: test_nti64: +; ALL: # BB#0: # %entry +; ALL-NEXT: movntiq %rsi, (%rdi) +; ALL-NEXT: retq +entry: + store i64 %X, i64* %ptr, align 8, !nontemporal !1 + ret void +} + +define void @test_ntfloat(float* nocapture %ptr, float %X) { +; SSE2-LABEL: test_ntfloat: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movss %xmm0, (%rdi) +; SSE2-NEXT: retq +; +; SSE4A-LABEL: test_ntfloat: +; SSE4A: # BB#0: # %entry +; SSE4A-NEXT: movntss %xmm0, (%rdi) +; SSE4A-NEXT: retq +; +; AVX-LABEL: test_ntfloat: +; AVX: # BB#0: # %entry +; AVX-NEXT: vmovss %xmm0, (%rdi) +; AVX-NEXT: retq +entry: + store float %X, float* %ptr, align 4, !nontemporal !1 + ret void +} + +define void @test_ntdouble(double* nocapture %ptr, double %X) { +; SSE2-LABEL: test_ntdouble: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movsd %xmm0, (%rdi) +; SSE2-NEXT: retq +; +; SSE4A-LABEL: test_ntdouble: +; SSE4A: # BB#0: # %entry +; SSE4A-NEXT: movntsd %xmm0, (%rdi) +; SSE4A-NEXT: retq +; +; AVX-LABEL: test_ntdouble: +; AVX: # BB#0: # %entry +; AVX-NEXT: vmovsd %xmm0, (%rdi) +; AVX-NEXT: retq +entry: + store double %X, double* %ptr, align 8, !nontemporal !1 + ret void +} + +define void @test_nt4xfloat(<4 x float>* nocapture %ptr, <4 x float> %X) { +; SSE-LABEL: test_nt4xfloat: +; SSE: # BB#0: # %entry +; SSE-NEXT: movntps %xmm0, (%rdi) +; SSE-NEXT: retq +; +; AVX-LABEL: test_nt4xfloat: +; AVX: # BB#0: # %entry +; AVX-NEXT: vmovntps %xmm0, (%rdi) +; AVX-NEXT: retq +entry: + store <4 x float> %X, <4 x float>* %ptr, align 16, !nontemporal !1 + ret void +} + +define void @test_nt2xdouble(<2 x double>* nocapture %ptr, <2 x double> %X) { +; SSE-LABEL: test_nt2xdouble: +; SSE: # BB#0: # %entry +; SSE-NEXT: movntpd %xmm0, (%rdi) +; SSE-NEXT: retq +; +; AVX-LABEL: test_nt2xdouble: +; AVX: # BB#0: # %entry +; AVX-NEXT: vmovntpd %xmm0, (%rdi) +; AVX-NEXT: retq +entry: + store <2 x double> %X, <2 x double>* %ptr, align 16, !nontemporal !1 + ret void +} + +define void @test_nt2xi64(<2 x i64>* nocapture %ptr, <2 x i64> %X) { +; SSE-LABEL: test_nt2xi64: +; SSE: # BB#0: # %entry +; SSE-NEXT: movntdq %xmm0, (%rdi) +; SSE-NEXT: retq +; +; AVX-LABEL: test_nt2xi64: +; AVX: # BB#0: # %entry +; AVX-NEXT: vmovntdq %xmm0, (%rdi) +; AVX-NEXT: retq +entry: + store <2 x i64> %X, <2 x i64>* %ptr, align 16, !nontemporal !1 + ret void +} + +!1 = !{i32 1} diff --git a/test/CodeGen/X86/fast-isel-stackcheck.ll b/test/CodeGen/X86/fast-isel-stackcheck.ll new file mode 100644 index 000000000000..3b7318fa77d9 --- /dev/null +++ b/test/CodeGen/X86/fast-isel-stackcheck.ll @@ -0,0 +1,44 @@ +; RUN: llc -o - %s | FileCheck %s +target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx" + +; selectiondag stack protector uses a GuardReg which the fast-isel stack +; protection code did not but the state was not reset properly. +; The optnone attribute on @bar forces fast-isel. + +; CHECK-LABEL: foo: +; CHECK: movq ___stack_chk_guard@GOTPCREL(%rip), %rax +; CHECK-NOT: movq ___stack_chk_guard@GOTPCREL(%rip), %rax +define void @foo() #0 { +entry: + %_tags = alloca [3 x i32], align 4 + ret void +} + +; CHECK-LABEL: bar: +; CHECK: movq ___stack_chk_guard@GOTPCREL(%rip), %rax +; CHECK: movq ___stack_chk_guard@GOTPCREL(%rip), %rax +define void @bar() #1 { +entry: + %vt = alloca [2 x double], align 16 + br i1 undef, label %cleanup.4091, label %for.cond.3850 + +unreachable: + unreachable + +for.cond.3850: + br i1 undef, label %land.rhs.3853, label %land.end.3857 + +land.rhs.3853: + br label %land.end.3857 + +land.end.3857: + %0 = phi i1 [ false, %for.cond.3850 ], [ false, %land.rhs.3853 ] + br i1 %0, label %unreachable, label %unreachable + +cleanup.4091: + ret void +} + +attributes #0 = { ssp } +attributes #1 = { noinline optnone ssp } diff --git a/test/CodeGen/X86/fast-isel-tls.ll b/test/CodeGen/X86/fast-isel-tls.ll index 18bb9c13ff01..0b7a5d9759d2 100644 --- a/test/CodeGen/X86/fast-isel-tls.ll +++ b/test/CodeGen/X86/fast-isel-tls.ll @@ -13,7 +13,7 @@ entry: ; CHECK: leal v@TLSGD ; CHECK: __tls_get_addr -@alias = internal alias i32* @v +@alias = internal alias i32, i32* @v define i32 @f_alias() nounwind { entry: %t = load i32, i32* @v diff --git a/test/CodeGen/X86/fdiv-combine.ll b/test/CodeGen/X86/fdiv-combine.ll index b65e9d01ab8b..d9d9ac401fb5 100644 --- a/test/CodeGen/X86/fdiv-combine.ll +++ b/test/CodeGen/X86/fdiv-combine.ll @@ -1,9 +1,11 @@ ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s -; Anything more than one division using a single divisor operand +; More than one 'arcp' division using a single divisor operand ; should be converted into a reciprocal and multiplication. -define float @div1_arcp(float %x, float %y, float %z) #0 { +; Don't do anything for just one division. + +define float @div1_arcp(float %x, float %y, float %z) { ; CHECK-LABEL: div1_arcp: ; CHECK: # BB#0: ; CHECK-NEXT: divss %xmm1, %xmm0 @@ -12,13 +14,15 @@ define float @div1_arcp(float %x, float %y, float %z) #0 { ret float %div1 } -define float @div2_arcp(float %x, float %y, float %z) #0 { -; CHECK-LABEL: div2_arcp: +; All math instructions are 'arcp', so optimize. + +define float @div2_arcp_all(float %x, float %y, float %z) { +; CHECK-LABEL: div2_arcp_all: ; CHECK: # BB#0: ; CHECK-NEXT: movss {{.*#+}} xmm3 = mem[0],zero,zero,zero ; CHECK-NEXT: divss %xmm2, %xmm3 -; CHECK-NEXT: mulss %xmm1, %xmm0 ; CHECK-NEXT: mulss %xmm3, %xmm0 +; CHECK-NEXT: mulss %xmm1, %xmm0 ; CHECK-NEXT: mulss %xmm3, %xmm0 ; CHECK-NEXT: retq %div1 = fdiv arcp float %x, %z @@ -27,10 +31,57 @@ define float @div2_arcp(float %x, float %y, float %z) #0 { ret float %div2 } +; The first division is not 'arcp', so do not optimize. + +define float @div2_arcp_partial1(float %x, float %y, float %z) { +; CHECK-LABEL: div2_arcp_partial1: +; CHECK: # BB#0: +; CHECK-NEXT: divss %xmm2, %xmm0 +; CHECK-NEXT: mulss %xmm1, %xmm0 +; CHECK-NEXT: divss %xmm2, %xmm0 +; CHECK-NEXT: retq + %div1 = fdiv float %x, %z + %mul = fmul arcp float %div1, %y + %div2 = fdiv arcp float %mul, %z + ret float %div2 +} + +; The second division is not 'arcp', so do not optimize. + +define float @div2_arcp_partial2(float %x, float %y, float %z) { +; CHECK-LABEL: div2_arcp_partial2: +; CHECK: # BB#0: +; CHECK-NEXT: divss %xmm2, %xmm0 +; CHECK-NEXT: mulss %xmm1, %xmm0 +; CHECK-NEXT: divss %xmm2, %xmm0 +; CHECK-NEXT: retq + %div1 = fdiv arcp float %x, %z + %mul = fmul arcp float %div1, %y + %div2 = fdiv float %mul, %z + ret float %div2 +} + +; The multiply is not 'arcp', but that does not prevent optimizing the divisions. + +define float @div2_arcp_partial3(float %x, float %y, float %z) { +; CHECK-LABEL: div2_arcp_partial3: +; CHECK: # BB#0: +; CHECK-NEXT: movss {{.*#+}} xmm3 = mem[0],zero,zero,zero +; CHECK-NEXT: divss %xmm2, %xmm3 +; CHECK-NEXT: mulss %xmm3, %xmm0 +; CHECK-NEXT: mulss %xmm1, %xmm0 +; CHECK-NEXT: mulss %xmm3, %xmm0 +; CHECK-NEXT: retq + %div1 = fdiv arcp float %x, %z + %mul = fmul float %div1, %y + %div2 = fdiv arcp float %mul, %z + ret float %div2 +} + ; If the reciprocal is already calculated, we should not ; generate an extra multiplication by 1.0. -define double @div3_arcp(double %x, double %y, double %z) #0 { +define double @div3_arcp(double %x, double %y, double %z) { ; CHECK-LABEL: div3_arcp: ; CHECK: # BB#0: ; CHECK-NEXT: movsd{{.*#+}} xmm2 = mem[0],zero @@ -44,7 +95,7 @@ define double @div3_arcp(double %x, double %y, double %z) #0 { ret double %ret } -define void @PR24141() #0 { +define void @PR24141() { ; CHECK-LABEL: PR24141: ; CHECK: callq ; CHECK-NEXT: divsd @@ -57,11 +108,9 @@ while.body: %call = call { double, double } @g(double %x.0) %xv0 = extractvalue { double, double } %call, 0 %xv1 = extractvalue { double, double } %call, 1 - %div = fdiv double %xv0, %xv1 + %div = fdiv arcp double %xv0, %xv1 br label %while.body } declare { double, double } @g(double) -; FIXME: If the backend understands 'arcp', then this attribute is unnecessary. -attributes #0 = { "unsafe-fp-math"="true" } diff --git a/test/CodeGen/X86/fdiv.ll b/test/CodeGen/X86/fdiv.ll index 0749682e2f68..226e6d269c3b 100644 --- a/test/CodeGen/X86/fdiv.ll +++ b/test/CodeGen/X86/fdiv.ll @@ -1,41 +1,69 @@ -; RUN: llc < %s -march=x86-64 -enable-unsafe-fp-math | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -enable-unsafe-fp-math | FileCheck %s define double @exact(double %x) { ; Exact division by a constant converted to multiplication. -; CHECK: @exact -; CHECK: mulsd +; CHECK-LABEL: exact: +; CHECK: # BB#0: +; CHECK-NEXT: mulsd {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq %div = fdiv double %x, 2.0 ret double %div } define double @inexact(double %x) { ; Inexact division by a constant converted to multiplication. -; CHECK: @inexact -; CHECK: mulsd - %div = fdiv double %x, 0x41DFFFFFFFC00000 +; CHECK-LABEL: inexact: +; CHECK: # BB#0: +; CHECK-NEXT: mulsd {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq + %div = fdiv double %x, 0x41DFFFFFFFC00000 ret double %div } define double @funky(double %x) { ; No conversion to multiplication if too funky. -; CHECK: @funky -; CHECK: divsd +; CHECK-LABEL: funky: +; CHECK: # BB#0: +; CHECK-NEXT: xorpd %xmm1, %xmm1 +; CHECK-NEXT: divsd %xmm1, %xmm0 +; CHECK-NEXT: retq %div = fdiv double %x, 0.0 ret double %div } define double @denormal1(double %x) { ; Don't generate multiplication by a denormal. -; CHECK: @denormal1 -; CHECK: divsd +; CHECK-LABEL: denormal1: +; CHECK: # BB#0: +; CHECK-NEXT: divsd {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq %div = fdiv double %x, 0x7FD0000000000001 ret double %div } define double @denormal2(double %x) { ; Don't generate multiplication by a denormal. -; CHECK: @denormal -; CHECK: divsd +; CHECK-LABEL: denormal2: +; CHECK: # BB#0: +; CHECK-NEXT: divsd {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq %div = fdiv double %x, 0x7FEFFFFFFFFFFFFF ret double %div } + +; Deleting the negates does not require unsafe-fp-math. + +define float @double_negative(float %x, float %y) #0 { +; CHECK-LABEL: double_negative: +; CHECK: # BB#0: +; CHECK-NEXT: divss %xmm1, %xmm0 +; CHECK-NEXT: retq + %neg1 = fsub float -0.0, %x + %neg2 = fsub float -0.0, %y + %div = fdiv float %neg1, %neg2 + ret float %div +} + +attributes #0 = { "unsafe-fp-math"="false" } + diff --git a/test/CodeGen/X86/fixup-lea.ll b/test/CodeGen/X86/fixup-lea.ll new file mode 100644 index 000000000000..1ddc099ffd62 --- /dev/null +++ b/test/CodeGen/X86/fixup-lea.ll @@ -0,0 +1,34 @@ +;RUN: llc < %s -march=x86 | FileCheck %s + +define void @foo(i32 inreg %dns) minsize { +entry: +; CHECK-LABEL: foo +; CHECK: dec + br label %for.body + +for.body: + %i.05 = phi i16 [ %dec, %for.body ], [ 0, %entry ] + %dec = add i16 %i.05, -1 + %conv = zext i16 %dec to i32 + %cmp = icmp slt i32 %conv, %dns + br i1 %cmp, label %for.body, label %for.end + +for.end: + ret void +} + +define void @bar(i32 inreg %dns) minsize { +entry: +; CHECK-LABEL: bar +; CHECK: inc + br label %for.body + +for.body: + %i.05 = phi i16 [ %inc, %for.body ], [ 0, %entry ] + %inc = add i16 %i.05, 1 + %conv = zext i16 %inc to i32 + %cmp = icmp slt i32 %conv, %dns + br i1 %cmp, label %for.body, label %for.end +for.end: + ret void +} diff --git a/test/CodeGen/X86/float-asmprint.ll b/test/CodeGen/X86/float-asmprint.ll index 5de9700fc064..0108430ee93e 100644 --- a/test/CodeGen/X86/float-asmprint.ll +++ b/test/CodeGen/X86/float-asmprint.ll @@ -9,6 +9,8 @@ @var64 = global double -0.0, align 8 @var32 = global float -0.0, align 4 @var16 = global half -0.0, align 2 +@var4f32 = global <4 x float> +@var4f16 = global <4 x half> ; CHECK: var128: ; CHECK-NEXT: .quad 0 # fp128 -0 @@ -39,3 +41,16 @@ ; CHECK-NEXT: .short 32768 # half -0 ; CHECK-NEXT: .size +; CHECK: var4f32: +; CHECK-NEXT: .long 2147483648 # float -0 +; CHECK-NEXT: .long 0 # float 0 +; CHECK-NEXT: .long 1065353216 # float 1 +; CHECK-NEXT: .long 1073741824 # float 2 +; CHECK-NEXT: .size + +; CHECK: var4f16: +; CHECK-NEXT: .short 32768 # half -0 +; CHECK-NEXT: .short 0 # half 0 +; CHECK-NEXT: .short 15360 # half 1 +; CHECK-NEXT: .short 16384 # half 2 +; CHECK-NEXT: .size diff --git a/test/CodeGen/X86/floor-soft-float.ll b/test/CodeGen/X86/floor-soft-float.ll index 7bb738513f54..3b28ecc6379d 100644 --- a/test/CodeGen/X86/floor-soft-float.ll +++ b/test/CodeGen/X86/floor-soft-float.ll @@ -6,7 +6,7 @@ target triple = "x86_64-unknown-linux-gnu" declare float @llvm.floor.f32(float) ; CHECK-SOFT-FLOAT: callq floorf -; CHECK-HARD-FLOAT: roundss $1, %xmm0, %xmm0 +; CHECK-HARD-FLOAT: roundss $9, %xmm0, %xmm0 define float @myfloor(float %a) { %val = tail call float @llvm.floor.f32(float %a) ret float %val diff --git a/test/CodeGen/X86/fma-commute-x86.ll b/test/CodeGen/X86/fma-commute-x86.ll new file mode 100644 index 000000000000..162a97ac025c --- /dev/null +++ b/test/CodeGen/X86/fma-commute-x86.ll @@ -0,0 +1,761 @@ +; RUN: llc < %s -mtriple=x86_64-pc-win32 -mcpu=core-avx2 | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-pc-win32 -mattr=+fma | FileCheck %s +; RUN: llc < %s -mcpu=bdver2 -mtriple=x86_64-pc-win32 -mattr=-fma4 | FileCheck %s + +attributes #0 = { nounwind } + +declare <4 x float> @llvm.x86.fma.vfmadd.ss(<4 x float>, <4 x float>, <4 x float>) nounwind readnone +define <4 x float> @test_x86_fmadd_baa_ss(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_baa_ss: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps {{\(%rdx\), %xmm0|\(%rcx\), %xmm1}} +; CHECK-NEXT: vmovaps {{\(%rdx\), %xmm0|\(%rcx\), %xmm1}} +; CHECK-NEXT: vfmadd213ss %xmm1, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfmadd.ss(<4 x float> %b, <4 x float> %a, <4 x float> %a) nounwind + ret <4 x float> %res +} + +define <4 x float> @test_x86_fmadd_aba_ss(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_aba_ss: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %xmm0 +; CHECK-NEXT: vfmadd132ss (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfmadd.ss(<4 x float> %a, <4 x float> %b, <4 x float> %a) nounwind + ret <4 x float> %res +} + +define <4 x float> @test_x86_fmadd_bba_ss(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_bba_ss: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rdx), %xmm0 +; CHECK-NEXT: vfmadd213ss (%rcx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfmadd.ss(<4 x float> %b, <4 x float> %b, <4 x float> %a) nounwind + ret <4 x float> %res +} + +declare <4 x float> @llvm.x86.fma.vfmadd.ps(<4 x float>, <4 x float>, <4 x float>) nounwind readnone +define <4 x float> @test_x86_fmadd_baa_ps(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_baa_ps: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %xmm0 +; CHECK-NEXT: vfmadd132ps (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfmadd.ps(<4 x float> %b, <4 x float> %a, <4 x float> %a) nounwind + ret <4 x float> %res +} + +define <4 x float> @test_x86_fmadd_aba_ps(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_aba_ps: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %xmm0 +; CHECK-NEXT: vfmadd231ps (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfmadd.ps(<4 x float> %a, <4 x float> %b, <4 x float> %a) nounwind + ret <4 x float> %res +} + +define <4 x float> @test_x86_fmadd_bba_ps(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_bba_ps: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rdx), %xmm0 +; CHECK-NEXT: vfmadd213ps (%rcx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfmadd.ps(<4 x float> %b, <4 x float> %b, <4 x float> %a) nounwind + ret <4 x float> %res +} + +declare <8 x float> @llvm.x86.fma.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>) nounwind readnone +define <8 x float> @test_x86_fmadd_baa_ps_y(<8 x float> %a, <8 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_baa_ps_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %ymm0 +; CHECK-NEXT: vfmadd132ps (%rdx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.fma.vfmadd.ps.256(<8 x float> %b, <8 x float> %a, <8 x float> %a) nounwind + ret <8 x float> %res +} + +define <8 x float> @test_x86_fmadd_aba_ps_y(<8 x float> %a, <8 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_aba_ps_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %ymm0 +; CHECK-NEXT: vfmadd231ps (%rdx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.fma.vfmadd.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %a) nounwind + ret <8 x float> %res +} + +define <8 x float> @test_x86_fmadd_bba_ps_y(<8 x float> %a, <8 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_bba_ps_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rdx), %ymm0 +; CHECK-NEXT: vfmadd213ps (%rcx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.fma.vfmadd.ps.256(<8 x float> %b, <8 x float> %b, <8 x float> %a) nounwind + ret <8 x float> %res +} + +declare <2 x double> @llvm.x86.fma.vfmadd.sd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone +define <2 x double> @test_x86_fmadd_baa_sd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_baa_sd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd {{\(%rdx\), %xmm0|\(%rcx\), %xmm1}} +; CHECK-NEXT: vmovapd {{\(%rdx\), %xmm0|\(%rcx\), %xmm1}} +; CHECK-NEXT: vfmadd213sd %xmm1, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfmadd.sd(<2 x double> %b, <2 x double> %a, <2 x double> %a) nounwind + ret <2 x double> %res +} + +define <2 x double> @test_x86_fmadd_aba_sd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_aba_sd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %xmm0 +; CHECK-NEXT: vfmadd132sd (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfmadd.sd(<2 x double> %a, <2 x double> %b, <2 x double> %a) nounwind + ret <2 x double> %res +} + +define <2 x double> @test_x86_fmadd_bba_sd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_bba_sd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rdx), %xmm0 +; CHECK-NEXT: vfmadd213sd (%rcx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfmadd.sd(<2 x double> %b, <2 x double> %b, <2 x double> %a) nounwind + ret <2 x double> %res +} + +declare <2 x double> @llvm.x86.fma.vfmadd.pd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone +define <2 x double> @test_x86_fmadd_baa_pd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_baa_pd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %xmm0 +; CHECK-NEXT: vfmadd132pd (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfmadd.pd(<2 x double> %b, <2 x double> %a, <2 x double> %a) nounwind + ret <2 x double> %res +} + +define <2 x double> @test_x86_fmadd_aba_pd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_aba_pd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %xmm0 +; CHECK-NEXT: vfmadd231pd (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfmadd.pd(<2 x double> %a, <2 x double> %b, <2 x double> %a) nounwind + ret <2 x double> %res +} + +define <2 x double> @test_x86_fmadd_bba_pd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_bba_pd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rdx), %xmm0 +; CHECK-NEXT: vfmadd213pd (%rcx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfmadd.pd(<2 x double> %b, <2 x double> %b, <2 x double> %a) nounwind + ret <2 x double> %res +} + +declare <4 x double> @llvm.x86.fma.vfmadd.pd.256(<4 x double>, <4 x double>, <4 x double>) nounwind readnone +define <4 x double> @test_x86_fmadd_baa_pd_y(<4 x double> %a, <4 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_baa_pd_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %ymm0 +; CHECK-NEXT: vfmadd132pd (%rdx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.fma.vfmadd.pd.256(<4 x double> %b, <4 x double> %a, <4 x double> %a) nounwind + ret <4 x double> %res +} + +define <4 x double> @test_x86_fmadd_aba_pd_y(<4 x double> %a, <4 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_aba_pd_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %ymm0 +; CHECK-NEXT: vfmadd231pd (%rdx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.fma.vfmadd.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %a) nounwind + ret <4 x double> %res +} + +define <4 x double> @test_x86_fmadd_bba_pd_y(<4 x double> %a, <4 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmadd_bba_pd_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rdx), %ymm0 +; CHECK-NEXT: vfmadd213pd (%rcx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.fma.vfmadd.pd.256(<4 x double> %b, <4 x double> %b, <4 x double> %a) nounwind + ret <4 x double> %res +} + + +declare <4 x float> @llvm.x86.fma.vfnmadd.ss(<4 x float>, <4 x float>, <4 x float>) nounwind readnone +define <4 x float> @test_x86_fnmadd_baa_ss(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_baa_ss: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps {{\(%rdx\), %xmm0|\(%rcx\), %xmm1}} +; CHECK-NEXT: vmovaps {{\(%rdx\), %xmm0|\(%rcx\), %xmm1}} +; CHECK-NEXT: vfnmadd213ss %xmm1, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfnmadd.ss(<4 x float> %b, <4 x float> %a, <4 x float> %a) nounwind + ret <4 x float> %res +} + +define <4 x float> @test_x86_fnmadd_aba_ss(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_aba_ss: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %xmm0 +; CHECK-NEXT: vfnmadd132ss (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfnmadd.ss(<4 x float> %a, <4 x float> %b, <4 x float> %a) nounwind + ret <4 x float> %res +} + +define <4 x float> @test_x86_fnmadd_bba_ss(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_bba_ss: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rdx), %xmm0 +; CHECK-NEXT: vfnmadd213ss (%rcx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfnmadd.ss(<4 x float> %b, <4 x float> %b, <4 x float> %a) nounwind + ret <4 x float> %res +} + +declare <4 x float> @llvm.x86.fma.vfnmadd.ps(<4 x float>, <4 x float>, <4 x float>) nounwind readnone +define <4 x float> @test_x86_fnmadd_baa_ps(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_baa_ps: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %xmm0 +; CHECK-NEXT: vfnmadd132ps (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfnmadd.ps(<4 x float> %b, <4 x float> %a, <4 x float> %a) nounwind + ret <4 x float> %res +} + +define <4 x float> @test_x86_fnmadd_aba_ps(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_aba_ps: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %xmm0 +; CHECK-NEXT: vfnmadd231ps (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfnmadd.ps(<4 x float> %a, <4 x float> %b, <4 x float> %a) nounwind + ret <4 x float> %res +} + +define <4 x float> @test_x86_fnmadd_bba_ps(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_bba_ps: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rdx), %xmm0 +; CHECK-NEXT: vfnmadd213ps (%rcx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfnmadd.ps(<4 x float> %b, <4 x float> %b, <4 x float> %a) nounwind + ret <4 x float> %res +} + +declare <8 x float> @llvm.x86.fma.vfnmadd.ps.256(<8 x float>, <8 x float>, <8 x float>) nounwind readnone +define <8 x float> @test_x86_fnmadd_baa_ps_y(<8 x float> %a, <8 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_baa_ps_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %ymm0 +; CHECK-NEXT: vfnmadd132ps (%rdx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.fma.vfnmadd.ps.256(<8 x float> %b, <8 x float> %a, <8 x float> %a) nounwind + ret <8 x float> %res +} + +define <8 x float> @test_x86_fnmadd_aba_ps_y(<8 x float> %a, <8 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_aba_ps_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %ymm0 +; CHECK-NEXT: vfnmadd231ps (%rdx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.fma.vfnmadd.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %a) nounwind + ret <8 x float> %res +} + +define <8 x float> @test_x86_fnmadd_bba_ps_y(<8 x float> %a, <8 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_bba_ps_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rdx), %ymm0 +; CHECK-NEXT: vfnmadd213ps (%rcx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.fma.vfnmadd.ps.256(<8 x float> %b, <8 x float> %b, <8 x float> %a) nounwind + ret <8 x float> %res +} + +declare <2 x double> @llvm.x86.fma.vfnmadd.sd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone +define <2 x double> @test_x86_fnmadd_baa_sd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_baa_sd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd {{\(%rdx\), %xmm0|\(%rcx\), %xmm1}} +; CHECK-NEXT: vmovapd {{\(%rdx\), %xmm0|\(%rcx\), %xmm1}} +; CHECK-NEXT: vfnmadd213sd %xmm1, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfnmadd.sd(<2 x double> %b, <2 x double> %a, <2 x double> %a) nounwind + ret <2 x double> %res +} + +define <2 x double> @test_x86_fnmadd_aba_sd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_aba_sd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %xmm0 +; CHECK-NEXT: vfnmadd132sd (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfnmadd.sd(<2 x double> %a, <2 x double> %b, <2 x double> %a) nounwind + ret <2 x double> %res +} + +define <2 x double> @test_x86_fnmadd_bba_sd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_bba_sd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rdx), %xmm0 +; CHECK-NEXT: vfnmadd213sd (%rcx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfnmadd.sd(<2 x double> %b, <2 x double> %b, <2 x double> %a) nounwind + ret <2 x double> %res +} + +declare <2 x double> @llvm.x86.fma.vfnmadd.pd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone +define <2 x double> @test_x86_fnmadd_baa_pd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_baa_pd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %xmm0 +; CHECK-NEXT: vfnmadd132pd (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfnmadd.pd(<2 x double> %b, <2 x double> %a, <2 x double> %a) nounwind + ret <2 x double> %res +} + +define <2 x double> @test_x86_fnmadd_aba_pd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_aba_pd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %xmm0 +; CHECK-NEXT: vfnmadd231pd (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfnmadd.pd(<2 x double> %a, <2 x double> %b, <2 x double> %a) nounwind + ret <2 x double> %res +} + +define <2 x double> @test_x86_fnmadd_bba_pd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_bba_pd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rdx), %xmm0 +; CHECK-NEXT: vfnmadd213pd (%rcx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfnmadd.pd(<2 x double> %b, <2 x double> %b, <2 x double> %a) nounwind + ret <2 x double> %res +} + +declare <4 x double> @llvm.x86.fma.vfnmadd.pd.256(<4 x double>, <4 x double>, <4 x double>) nounwind readnone +define <4 x double> @test_x86_fnmadd_baa_pd_y(<4 x double> %a, <4 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_baa_pd_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %ymm0 +; CHECK-NEXT: vfnmadd132pd (%rdx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.fma.vfnmadd.pd.256(<4 x double> %b, <4 x double> %a, <4 x double> %a) nounwind + ret <4 x double> %res +} + +define <4 x double> @test_x86_fnmadd_aba_pd_y(<4 x double> %a, <4 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_aba_pd_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %ymm0 +; CHECK-NEXT: vfnmadd231pd (%rdx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.fma.vfnmadd.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %a) nounwind + ret <4 x double> %res +} + +define <4 x double> @test_x86_fnmadd_bba_pd_y(<4 x double> %a, <4 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmadd_bba_pd_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rdx), %ymm0 +; CHECK-NEXT: vfnmadd213pd (%rcx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.fma.vfnmadd.pd.256(<4 x double> %b, <4 x double> %b, <4 x double> %a) nounwind + ret <4 x double> %res +} + + +declare <4 x float> @llvm.x86.fma.vfmsub.ss(<4 x float>, <4 x float>, <4 x float>) nounwind readnone +define <4 x float> @test_x86_fmsub_baa_ss(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_baa_ss: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps {{\(%rdx\), %xmm0|\(%rcx\), %xmm1}} +; CHECK-NEXT: vmovaps {{\(%rdx\), %xmm0|\(%rcx\), %xmm1}} +; CHECK-NEXT: vfmsub213ss %xmm1, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfmsub.ss(<4 x float> %b, <4 x float> %a, <4 x float> %a) nounwind + ret <4 x float> %res +} + +define <4 x float> @test_x86_fmsub_aba_ss(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_aba_ss: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %xmm0 +; CHECK-NEXT: vfmsub132ss (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfmsub.ss(<4 x float> %a, <4 x float> %b, <4 x float> %a) nounwind + ret <4 x float> %res +} + +define <4 x float> @test_x86_fmsub_bba_ss(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_bba_ss: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rdx), %xmm0 +; CHECK-NEXT: vfmsub213ss (%rcx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfmsub.ss(<4 x float> %b, <4 x float> %b, <4 x float> %a) nounwind + ret <4 x float> %res +} + +declare <4 x float> @llvm.x86.fma.vfmsub.ps(<4 x float>, <4 x float>, <4 x float>) nounwind readnone +define <4 x float> @test_x86_fmsub_baa_ps(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_baa_ps: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %xmm0 +; CHECK-NEXT: vfmsub132ps (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfmsub.ps(<4 x float> %b, <4 x float> %a, <4 x float> %a) nounwind + ret <4 x float> %res +} + +define <4 x float> @test_x86_fmsub_aba_ps(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_aba_ps: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %xmm0 +; CHECK-NEXT: vfmsub231ps (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfmsub.ps(<4 x float> %a, <4 x float> %b, <4 x float> %a) nounwind + ret <4 x float> %res +} + +define <4 x float> @test_x86_fmsub_bba_ps(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_bba_ps: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rdx), %xmm0 +; CHECK-NEXT: vfmsub213ps (%rcx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfmsub.ps(<4 x float> %b, <4 x float> %b, <4 x float> %a) nounwind + ret <4 x float> %res +} + +declare <8 x float> @llvm.x86.fma.vfmsub.ps.256(<8 x float>, <8 x float>, <8 x float>) nounwind readnone +define <8 x float> @test_x86_fmsub_baa_ps_y(<8 x float> %a, <8 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_baa_ps_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %ymm0 +; CHECK-NEXT: vfmsub132ps (%rdx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.fma.vfmsub.ps.256(<8 x float> %b, <8 x float> %a, <8 x float> %a) nounwind + ret <8 x float> %res +} + +define <8 x float> @test_x86_fmsub_aba_ps_y(<8 x float> %a, <8 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_aba_ps_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %ymm0 +; CHECK-NEXT: vfmsub231ps (%rdx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.fma.vfmsub.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %a) nounwind + ret <8 x float> %res +} + +define <8 x float> @test_x86_fmsub_bba_ps_y(<8 x float> %a, <8 x float> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_bba_ps_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rdx), %ymm0 +; CHECK-NEXT: vfmsub213ps (%rcx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.fma.vfmsub.ps.256(<8 x float> %b, <8 x float> %b, <8 x float> %a) nounwind + ret <8 x float> %res +} + +declare <2 x double> @llvm.x86.fma.vfmsub.sd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone +define <2 x double> @test_x86_fmsub_baa_sd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_baa_sd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd {{\(%rdx\), %xmm0|\(%rcx\), %xmm1}} +; CHECK-NEXT: vmovapd {{\(%rdx\), %xmm0|\(%rcx\), %xmm1}} +; CHECK-NEXT: vfmsub213sd %xmm1, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfmsub.sd(<2 x double> %b, <2 x double> %a, <2 x double> %a) nounwind + ret <2 x double> %res +} + +define <2 x double> @test_x86_fmsub_aba_sd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_aba_sd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %xmm0 +; CHECK-NEXT: vfmsub132sd (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfmsub.sd(<2 x double> %a, <2 x double> %b, <2 x double> %a) nounwind + ret <2 x double> %res +} + +define <2 x double> @test_x86_fmsub_bba_sd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_bba_sd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rdx), %xmm0 +; CHECK-NEXT: vfmsub213sd (%rcx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfmsub.sd(<2 x double> %b, <2 x double> %b, <2 x double> %a) nounwind + ret <2 x double> %res +} + +declare <2 x double> @llvm.x86.fma.vfmsub.pd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone +define <2 x double> @test_x86_fmsub_baa_pd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_baa_pd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %xmm0 +; CHECK-NEXT: vfmsub132pd (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfmsub.pd(<2 x double> %b, <2 x double> %a, <2 x double> %a) nounwind + ret <2 x double> %res +} + +define <2 x double> @test_x86_fmsub_aba_pd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_aba_pd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %xmm0 +; CHECK-NEXT: vfmsub231pd (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfmsub.pd(<2 x double> %a, <2 x double> %b, <2 x double> %a) nounwind + ret <2 x double> %res +} + +define <2 x double> @test_x86_fmsub_bba_pd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_bba_pd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rdx), %xmm0 +; CHECK-NEXT: vfmsub213pd (%rcx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfmsub.pd(<2 x double> %b, <2 x double> %b, <2 x double> %a) nounwind + ret <2 x double> %res +} + +declare <4 x double> @llvm.x86.fma.vfmsub.pd.256(<4 x double>, <4 x double>, <4 x double>) nounwind readnone +define <4 x double> @test_x86_fmsub_baa_pd_y(<4 x double> %a, <4 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_baa_pd_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %ymm0 +; CHECK-NEXT: vfmsub132pd (%rdx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.fma.vfmsub.pd.256(<4 x double> %b, <4 x double> %a, <4 x double> %a) nounwind + ret <4 x double> %res +} + +define <4 x double> @test_x86_fmsub_aba_pd_y(<4 x double> %a, <4 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_aba_pd_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %ymm0 +; CHECK-NEXT: vfmsub231pd (%rdx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.fma.vfmsub.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %a) nounwind + ret <4 x double> %res +} + +define <4 x double> @test_x86_fmsub_bba_pd_y(<4 x double> %a, <4 x double> %b) #0 { +; CHECK-LABEL: test_x86_fmsub_bba_pd_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rdx), %ymm0 +; CHECK-NEXT: vfmsub213pd (%rcx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.fma.vfmsub.pd.256(<4 x double> %b, <4 x double> %b, <4 x double> %a) nounwind + ret <4 x double> %res +} + + +declare <4 x float> @llvm.x86.fma.vfnmsub.ss(<4 x float>, <4 x float>, <4 x float>) nounwind readnone +define <4 x float> @test_x86_fnmsub_baa_ss(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_baa_ss: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps {{\(%rdx\), %xmm0|\(%rcx\), %xmm1}} +; CHECK-NEXT: vmovaps {{\(%rdx\), %xmm0|\(%rcx\), %xmm1}} +; CHECK-NEXT: vfnmsub213ss %xmm1, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfnmsub.ss(<4 x float> %b, <4 x float> %a, <4 x float> %a) nounwind + ret <4 x float> %res +} + +define <4 x float> @test_x86_fnmsub_aba_ss(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_aba_ss: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %xmm0 +; CHECK-NEXT: vfnmsub132ss (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfnmsub.ss(<4 x float> %a, <4 x float> %b, <4 x float> %a) nounwind + ret <4 x float> %res +} + +define <4 x float> @test_x86_fnmsub_bba_ss(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_bba_ss: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rdx), %xmm0 +; CHECK-NEXT: vfnmsub213ss (%rcx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfnmsub.ss(<4 x float> %b, <4 x float> %b, <4 x float> %a) nounwind + ret <4 x float> %res +} + +declare <4 x float> @llvm.x86.fma.vfnmsub.ps(<4 x float>, <4 x float>, <4 x float>) nounwind readnone +define <4 x float> @test_x86_fnmsub_baa_ps(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_baa_ps: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %xmm0 +; CHECK-NEXT: vfnmsub132ps (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfnmsub.ps(<4 x float> %b, <4 x float> %a, <4 x float> %a) nounwind + ret <4 x float> %res +} + +define <4 x float> @test_x86_fnmsub_aba_ps(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_aba_ps: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %xmm0 +; CHECK-NEXT: vfnmsub231ps (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfnmsub.ps(<4 x float> %a, <4 x float> %b, <4 x float> %a) nounwind + ret <4 x float> %res +} + +define <4 x float> @test_x86_fnmsub_bba_ps(<4 x float> %a, <4 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_bba_ps: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rdx), %xmm0 +; CHECK-NEXT: vfnmsub213ps (%rcx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfnmsub.ps(<4 x float> %b, <4 x float> %b, <4 x float> %a) nounwind + ret <4 x float> %res +} + +declare <8 x float> @llvm.x86.fma.vfnmsub.ps.256(<8 x float>, <8 x float>, <8 x float>) nounwind readnone +define <8 x float> @test_x86_fnmsub_baa_ps_y(<8 x float> %a, <8 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_baa_ps_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %ymm0 +; CHECK-NEXT: vfnmsub132ps (%rdx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.fma.vfnmsub.ps.256(<8 x float> %b, <8 x float> %a, <8 x float> %a) nounwind + ret <8 x float> %res +} + +define <8 x float> @test_x86_fnmsub_aba_ps_y(<8 x float> %a, <8 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_aba_ps_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rcx), %ymm0 +; CHECK-NEXT: vfnmsub231ps (%rdx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.fma.vfnmsub.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %a) nounwind + ret <8 x float> %res +} + +define <8 x float> @test_x86_fnmsub_bba_ps_y(<8 x float> %a, <8 x float> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_bba_ps_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps (%rdx), %ymm0 +; CHECK-NEXT: vfnmsub213ps (%rcx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <8 x float> @llvm.x86.fma.vfnmsub.ps.256(<8 x float> %b, <8 x float> %b, <8 x float> %a) nounwind + ret <8 x float> %res +} + +declare <2 x double> @llvm.x86.fma.vfnmsub.sd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone +define <2 x double> @test_x86_fnmsub_baa_sd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_baa_sd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd {{\(%rdx\), %xmm0|\(%rcx\), %xmm1}} +; CHECK-NEXT: vmovapd {{\(%rdx\), %xmm0|\(%rcx\), %xmm1}} +; CHECK-NEXT: vfnmsub213sd %xmm1, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfnmsub.sd(<2 x double> %b, <2 x double> %a, <2 x double> %a) nounwind + ret <2 x double> %res +} + +define <2 x double> @test_x86_fnmsub_aba_sd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_aba_sd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %xmm0 +; CHECK-NEXT: vfnmsub132sd (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfnmsub.sd(<2 x double> %a, <2 x double> %b, <2 x double> %a) nounwind + ret <2 x double> %res +} + +define <2 x double> @test_x86_fnmsub_bba_sd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_bba_sd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rdx), %xmm0 +; CHECK-NEXT: vfnmsub213sd (%rcx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfnmsub.sd(<2 x double> %b, <2 x double> %b, <2 x double> %a) nounwind + ret <2 x double> %res +} + +declare <2 x double> @llvm.x86.fma.vfnmsub.pd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone +define <2 x double> @test_x86_fnmsub_baa_pd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_baa_pd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %xmm0 +; CHECK-NEXT: vfnmsub132pd (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfnmsub.pd(<2 x double> %b, <2 x double> %a, <2 x double> %a) nounwind + ret <2 x double> %res +} + +define <2 x double> @test_x86_fnmsub_aba_pd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_aba_pd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %xmm0 +; CHECK-NEXT: vfnmsub231pd (%rdx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfnmsub.pd(<2 x double> %a, <2 x double> %b, <2 x double> %a) nounwind + ret <2 x double> %res +} + +define <2 x double> @test_x86_fnmsub_bba_pd(<2 x double> %a, <2 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_bba_pd: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rdx), %xmm0 +; CHECK-NEXT: vfnmsub213pd (%rcx), %xmm0, %xmm0 +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfnmsub.pd(<2 x double> %b, <2 x double> %b, <2 x double> %a) nounwind + ret <2 x double> %res +} + +declare <4 x double> @llvm.x86.fma.vfnmsub.pd.256(<4 x double>, <4 x double>, <4 x double>) nounwind readnone +define <4 x double> @test_x86_fnmsub_baa_pd_y(<4 x double> %a, <4 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_baa_pd_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %ymm0 +; CHECK-NEXT: vfnmsub132pd (%rdx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.fma.vfnmsub.pd.256(<4 x double> %b, <4 x double> %a, <4 x double> %a) nounwind + ret <4 x double> %res +} + +define <4 x double> @test_x86_fnmsub_aba_pd_y(<4 x double> %a, <4 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_aba_pd_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rcx), %ymm0 +; CHECK-NEXT: vfnmsub231pd (%rdx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.fma.vfnmsub.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %a) nounwind + ret <4 x double> %res +} + +define <4 x double> @test_x86_fnmsub_bba_pd_y(<4 x double> %a, <4 x double> %b) #0 { +; CHECK-LABEL: test_x86_fnmsub_bba_pd_y: +; CHECK: # BB#0: +; CHECK-NEXT: vmovapd (%rdx), %ymm0 +; CHECK-NEXT: vfnmsub213pd (%rcx), %ymm0, %ymm0 +; CHECK-NEXT: retq + %res = call <4 x double> @llvm.x86.fma.vfnmsub.pd.256(<4 x double> %b, <4 x double> %b, <4 x double> %a) nounwind + ret <4 x double> %res +} + diff --git a/test/CodeGen/X86/fma-do-not-commute.ll b/test/CodeGen/X86/fma-do-not-commute.ll index 1f6a19cfff83..89be0795d206 100644 --- a/test/CodeGen/X86/fma-do-not-commute.ll +++ b/test/CodeGen/X86/fma-do-not-commute.ll @@ -6,7 +6,7 @@ target triple = "x86_64-apple-macosx" ; CHECK-LABEL: test1: ; %arg lives in xmm0 and it shouldn't be redefined until it is used in the FMA. -; CHECK-NOT {{.*}}, %xmm0 +; CHECK-NOT: {{.*}}, %xmm0 ; %addr lives in rdi. ; %addr2 lives in rsi. ; CHECK: vmovss (%rsi), [[ADDR2:%xmm[0-9]+]] diff --git a/test/CodeGen/X86/fma-intrinsics-phi-213-to-231.ll b/test/CodeGen/X86/fma-intrinsics-phi-213-to-231.ll index f7d0cdf3c65a..8d0318bb93e0 100644 --- a/test/CodeGen/X86/fma-intrinsics-phi-213-to-231.ll +++ b/test/CodeGen/X86/fma-intrinsics-phi-213-to-231.ll @@ -1,8 +1,337 @@ ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fma | FileCheck %s -; CHECK-LABEL: fmaddsubpd_loop: -; CHECK: vfmaddsub231pd %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}} -define <4 x double> @fmaddsubpd_loop(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) { +; CHECK-LABEL: fmaddsubpd_loop_128: +; CHECK: vfmaddsub231pd %xmm1, %xmm0, %xmm2 +; CHECK: vmovaps %xmm2, %xmm0 +; CHECK-NEXT: retq +define <2 x double> @fmaddsubpd_loop_128(i32 %iter, <2 x double> %a, <2 x double> %b, <2 x double> %c) { +entry: + br label %for.cond + +for.cond: + %c.addr.0 = phi <2 x double> [ %c, %entry ], [ %0, %for.inc ] + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ] + %cmp = icmp slt i32 %i.0, %iter + br i1 %cmp, label %for.body, label %for.end + +for.body: + br label %for.inc + +for.inc: + %0 = call <2 x double> @llvm.x86.fma.vfmaddsub.pd(<2 x double> %a, <2 x double> %b, <2 x double> %c.addr.0) + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: + ret <2 x double> %c.addr.0 +} + +; CHECK-LABEL: fmsubaddpd_loop_128: +; CHECK: vfmsubadd231pd %xmm1, %xmm0, %xmm2 +; CHECK: vmovaps %xmm2, %xmm0 +; CHECK-NEXT: retq +define <2 x double> @fmsubaddpd_loop_128(i32 %iter, <2 x double> %a, <2 x double> %b, <2 x double> %c) { +entry: + br label %for.cond + +for.cond: + %c.addr.0 = phi <2 x double> [ %c, %entry ], [ %0, %for.inc ] + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ] + %cmp = icmp slt i32 %i.0, %iter + br i1 %cmp, label %for.body, label %for.end + +for.body: + br label %for.inc + +for.inc: + %0 = call <2 x double> @llvm.x86.fma.vfmsubadd.pd(<2 x double> %a, <2 x double> %b, <2 x double> %c.addr.0) + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: + ret <2 x double> %c.addr.0 +} + +; CHECK-LABEL: fmaddpd_loop_128: +; CHECK: vfmadd231pd %xmm1, %xmm0, %xmm2 +; CHECK: vmovaps %xmm2, %xmm0 +; CHECK-NEXT: retq +define <2 x double> @fmaddpd_loop_128(i32 %iter, <2 x double> %a, <2 x double> %b, <2 x double> %c) { +entry: + br label %for.cond + +for.cond: + %c.addr.0 = phi <2 x double> [ %c, %entry ], [ %0, %for.inc ] + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ] + %cmp = icmp slt i32 %i.0, %iter + br i1 %cmp, label %for.body, label %for.end + +for.body: + br label %for.inc + +for.inc: + %0 = call <2 x double> @llvm.x86.fma.vfmadd.pd(<2 x double> %a, <2 x double> %b, <2 x double> %c.addr.0) + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: + ret <2 x double> %c.addr.0 +} + +; CHECK-LABEL: fmsubpd_loop_128: +; CHECK: vfmsub231pd %xmm1, %xmm0, %xmm2 +; CHECK: vmovaps %xmm2, %xmm0 +; CHECK-NEXT: retq +define <2 x double> @fmsubpd_loop_128(i32 %iter, <2 x double> %a, <2 x double> %b, <2 x double> %c) { +entry: + br label %for.cond + +for.cond: + %c.addr.0 = phi <2 x double> [ %c, %entry ], [ %0, %for.inc ] + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ] + %cmp = icmp slt i32 %i.0, %iter + br i1 %cmp, label %for.body, label %for.end + +for.body: + br label %for.inc + +for.inc: + %0 = call <2 x double> @llvm.x86.fma.vfmsub.pd(<2 x double> %a, <2 x double> %b, <2 x double> %c.addr.0) + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: + ret <2 x double> %c.addr.0 +} + +; CHECK-LABEL: fnmaddpd_loop_128: +; CHECK: vfnmadd231pd %xmm1, %xmm0, %xmm2 +; CHECK: vmovaps %xmm2, %xmm0 +; CHECK-NEXT: retq +define <2 x double> @fnmaddpd_loop_128(i32 %iter, <2 x double> %a, <2 x double> %b, <2 x double> %c) { +entry: + br label %for.cond + +for.cond: + %c.addr.0 = phi <2 x double> [ %c, %entry ], [ %0, %for.inc ] + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ] + %cmp = icmp slt i32 %i.0, %iter + br i1 %cmp, label %for.body, label %for.end + +for.body: + br label %for.inc + +for.inc: + %0 = call <2 x double> @llvm.x86.fma.vfnmadd.pd(<2 x double> %a, <2 x double> %b, <2 x double> %c.addr.0) + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: + ret <2 x double> %c.addr.0 +} + +; CHECK-LABEL: fnmsubpd_loop_128: +; CHECK: vfnmsub231pd %xmm1, %xmm0, %xmm2 +; CHECK: vmovaps %xmm2, %xmm0 +; CHECK-NEXT: retq +define <2 x double> @fnmsubpd_loop_128(i32 %iter, <2 x double> %a, <2 x double> %b, <2 x double> %c) { +entry: + br label %for.cond + +for.cond: + %c.addr.0 = phi <2 x double> [ %c, %entry ], [ %0, %for.inc ] + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ] + %cmp = icmp slt i32 %i.0, %iter + br i1 %cmp, label %for.body, label %for.end + +for.body: + br label %for.inc + +for.inc: + %0 = call <2 x double> @llvm.x86.fma.vfnmsub.pd(<2 x double> %a, <2 x double> %b, <2 x double> %c.addr.0) + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: + ret <2 x double> %c.addr.0 +} + +declare <2 x double> @llvm.x86.fma.vfmaddsub.pd(<2 x double>, <2 x double>, <2 x double>) +declare <2 x double> @llvm.x86.fma.vfmsubadd.pd(<2 x double>, <2 x double>, <2 x double>) +declare <2 x double> @llvm.x86.fma.vfmadd.pd(<2 x double>, <2 x double>, <2 x double>) +declare <2 x double> @llvm.x86.fma.vfmsub.pd(<2 x double>, <2 x double>, <2 x double>) +declare <2 x double> @llvm.x86.fma.vfnmadd.pd(<2 x double>, <2 x double>, <2 x double>) +declare <2 x double> @llvm.x86.fma.vfnmsub.pd(<2 x double>, <2 x double>, <2 x double>) + + +; CHECK-LABEL: fmaddsubps_loop_128: +; CHECK: vfmaddsub231ps %xmm1, %xmm0, %xmm2 +; CHECK: vmovaps %xmm2, %xmm0 +; CHECK-NEXT: retq +define <4 x float> @fmaddsubps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %b, <4 x float> %c) { +entry: + br label %for.cond + +for.cond: + %c.addr.0 = phi <4 x float> [ %c, %entry ], [ %0, %for.inc ] + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ] + %cmp = icmp slt i32 %i.0, %iter + br i1 %cmp, label %for.body, label %for.end + +for.body: + br label %for.inc + +for.inc: + %0 = call <4 x float> @llvm.x86.fma.vfmaddsub.ps(<4 x float> %a, <4 x float> %b, <4 x float> %c.addr.0) + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: + ret <4 x float> %c.addr.0 +} + +; CHECK-LABEL: fmsubaddps_loop_128: +; CHECK: vfmsubadd231ps %xmm1, %xmm0, %xmm2 +; CHECK: vmovaps %xmm2, %xmm0 +; CHECK-NEXT: retq +define <4 x float> @fmsubaddps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %b, <4 x float> %c) { +entry: + br label %for.cond + +for.cond: + %c.addr.0 = phi <4 x float> [ %c, %entry ], [ %0, %for.inc ] + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ] + %cmp = icmp slt i32 %i.0, %iter + br i1 %cmp, label %for.body, label %for.end + +for.body: + br label %for.inc + +for.inc: + %0 = call <4 x float> @llvm.x86.fma.vfmsubadd.ps(<4 x float> %a, <4 x float> %b, <4 x float> %c.addr.0) + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: + ret <4 x float> %c.addr.0 +} + +; CHECK-LABEL: fmaddps_loop_128: +; CHECK: vfmadd231ps %xmm1, %xmm0, %xmm2 +; CHECK: vmovaps %xmm2, %xmm0 +; CHECK-NEXT: retq +define <4 x float> @fmaddps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %b, <4 x float> %c) { +entry: + br label %for.cond + +for.cond: + %c.addr.0 = phi <4 x float> [ %c, %entry ], [ %0, %for.inc ] + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ] + %cmp = icmp slt i32 %i.0, %iter + br i1 %cmp, label %for.body, label %for.end + +for.body: + br label %for.inc + +for.inc: + %0 = call <4 x float> @llvm.x86.fma.vfmadd.ps(<4 x float> %a, <4 x float> %b, <4 x float> %c.addr.0) + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: + ret <4 x float> %c.addr.0 +} + +; CHECK-LABEL: fmsubps_loop_128: +; CHECK: vfmsub231ps %xmm1, %xmm0, %xmm2 +; CHECK: vmovaps %xmm2, %xmm0 +; CHECK-NEXT: retq +define <4 x float> @fmsubps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %b, <4 x float> %c) { +entry: + br label %for.cond + +for.cond: + %c.addr.0 = phi <4 x float> [ %c, %entry ], [ %0, %for.inc ] + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ] + %cmp = icmp slt i32 %i.0, %iter + br i1 %cmp, label %for.body, label %for.end + +for.body: + br label %for.inc + +for.inc: + %0 = call <4 x float> @llvm.x86.fma.vfmsub.ps(<4 x float> %a, <4 x float> %b, <4 x float> %c.addr.0) + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: + ret <4 x float> %c.addr.0 +} + +; CHECK-LABEL: fnmaddps_loop_128: +; CHECK: vfnmadd231ps %xmm1, %xmm0, %xmm2 +; CHECK: vmovaps %xmm2, %xmm0 +; CHECK-NEXT: retq +define <4 x float> @fnmaddps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %b, <4 x float> %c) { +entry: + br label %for.cond + +for.cond: + %c.addr.0 = phi <4 x float> [ %c, %entry ], [ %0, %for.inc ] + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ] + %cmp = icmp slt i32 %i.0, %iter + br i1 %cmp, label %for.body, label %for.end + +for.body: + br label %for.inc + +for.inc: + %0 = call <4 x float> @llvm.x86.fma.vfnmadd.ps(<4 x float> %a, <4 x float> %b, <4 x float> %c.addr.0) + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: + ret <4 x float> %c.addr.0 +} + +; CHECK-LABEL: fnmsubps_loop_128: +; CHECK: vfnmsub231ps %xmm1, %xmm0, %xmm2 +; CHECK: vmovaps %xmm2, %xmm0 +; CHECK-NEXT: retq +define <4 x float> @fnmsubps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %b, <4 x float> %c) { +entry: + br label %for.cond + +for.cond: + %c.addr.0 = phi <4 x float> [ %c, %entry ], [ %0, %for.inc ] + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ] + %cmp = icmp slt i32 %i.0, %iter + br i1 %cmp, label %for.body, label %for.end + +for.body: + br label %for.inc + +for.inc: + %0 = call <4 x float> @llvm.x86.fma.vfnmsub.ps(<4 x float> %a, <4 x float> %b, <4 x float> %c.addr.0) + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: + ret <4 x float> %c.addr.0 +} + +declare <4 x float> @llvm.x86.fma.vfmaddsub.ps(<4 x float>, <4 x float>, <4 x float>) +declare <4 x float> @llvm.x86.fma.vfmsubadd.ps(<4 x float>, <4 x float>, <4 x float>) +declare <4 x float> @llvm.x86.fma.vfmadd.ps(<4 x float>, <4 x float>, <4 x float>) +declare <4 x float> @llvm.x86.fma.vfmsub.ps(<4 x float>, <4 x float>, <4 x float>) +declare <4 x float> @llvm.x86.fma.vfnmadd.ps(<4 x float>, <4 x float>, <4 x float>) +declare <4 x float> @llvm.x86.fma.vfnmsub.ps(<4 x float>, <4 x float>, <4 x float>) + +; CHECK-LABEL: fmaddsubpd_loop_256: +; CHECK: vfmaddsub231pd %ymm1, %ymm0, %ymm2 +; CHECK: vmovaps %ymm2, %ymm0 +; CHECK-NEXT: retq +define <4 x double> @fmaddsubpd_loop_256(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) { entry: br label %for.cond @@ -24,9 +353,11 @@ for.end: ret <4 x double> %c.addr.0 } -; CHECK-LABEL: fmsubaddpd_loop: -; CHECK: vfmsubadd231pd %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}} -define <4 x double> @fmsubaddpd_loop(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) { +; CHECK-LABEL: fmsubaddpd_loop_256: +; CHECK: vfmsubadd231pd %ymm1, %ymm0, %ymm2 +; CHECK: vmovaps %ymm2, %ymm0 +; CHECK-NEXT: retq +define <4 x double> @fmsubaddpd_loop_256(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) { entry: br label %for.cond @@ -48,9 +379,11 @@ for.end: ret <4 x double> %c.addr.0 } -; CHECK-LABEL: fmaddpd_loop: -; CHECK: vfmadd231pd %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}} -define <4 x double> @fmaddpd_loop(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) { +; CHECK-LABEL: fmaddpd_loop_256: +; CHECK: vfmadd231pd %ymm1, %ymm0, %ymm2 +; CHECK: vmovaps %ymm2, %ymm0 +; CHECK-NEXT: retq +define <4 x double> @fmaddpd_loop_256(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) { entry: br label %for.cond @@ -72,9 +405,11 @@ for.end: ret <4 x double> %c.addr.0 } -; CHECK-LABEL: fmsubpd_loop: -; CHECK: vfmsub231pd %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}} -define <4 x double> @fmsubpd_loop(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) { +; CHECK-LABEL: fmsubpd_loop_256: +; CHECK: vfmsub231pd %ymm1, %ymm0, %ymm2 +; CHECK: vmovaps %ymm2, %ymm0 +; CHECK-NEXT: retq +define <4 x double> @fmsubpd_loop_256(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) { entry: br label %for.cond @@ -96,15 +431,71 @@ for.end: ret <4 x double> %c.addr.0 } +; CHECK-LABEL: fnmaddpd_loop_256: +; CHECK: vfnmadd231pd %ymm1, %ymm0, %ymm2 +; CHECK: vmovaps %ymm2, %ymm0 +; CHECK-NEXT: retq +define <4 x double> @fnmaddpd_loop_256(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) { +entry: + br label %for.cond + +for.cond: + %c.addr.0 = phi <4 x double> [ %c, %entry ], [ %0, %for.inc ] + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ] + %cmp = icmp slt i32 %i.0, %iter + br i1 %cmp, label %for.body, label %for.end + +for.body: + br label %for.inc + +for.inc: + %0 = call <4 x double> @llvm.x86.fma.vfnmadd.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c.addr.0) + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: + ret <4 x double> %c.addr.0 +} + +; CHECK-LABEL: fnmsubpd_loop_256: +; CHECK: vfnmsub231pd %ymm1, %ymm0, %ymm2 +; CHECK: vmovaps %ymm2, %ymm0 +; CHECK-NEXT: retq +define <4 x double> @fnmsubpd_loop_256(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) { +entry: + br label %for.cond + +for.cond: + %c.addr.0 = phi <4 x double> [ %c, %entry ], [ %0, %for.inc ] + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ] + %cmp = icmp slt i32 %i.0, %iter + br i1 %cmp, label %for.body, label %for.end + +for.body: + br label %for.inc + +for.inc: + %0 = call <4 x double> @llvm.x86.fma.vfnmsub.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c.addr.0) + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: + ret <4 x double> %c.addr.0 +} + declare <4 x double> @llvm.x86.fma.vfmaddsub.pd.256(<4 x double>, <4 x double>, <4 x double>) declare <4 x double> @llvm.x86.fma.vfmsubadd.pd.256(<4 x double>, <4 x double>, <4 x double>) declare <4 x double> @llvm.x86.fma.vfmadd.pd.256(<4 x double>, <4 x double>, <4 x double>) declare <4 x double> @llvm.x86.fma.vfmsub.pd.256(<4 x double>, <4 x double>, <4 x double>) +declare <4 x double> @llvm.x86.fma.vfnmadd.pd.256(<4 x double>, <4 x double>, <4 x double>) +declare <4 x double> @llvm.x86.fma.vfnmsub.pd.256(<4 x double>, <4 x double>, <4 x double>) -; CHECK-LABEL: fmaddsubps_loop: -; CHECK: vfmaddsub231ps %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}} -define <8 x float> @fmaddsubps_loop(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) { +; CHECK-LABEL: fmaddsubps_loop_256: +; CHECK: vfmaddsub231ps %ymm1, %ymm0, %ymm2 +; CHECK: vmovaps %ymm2, %ymm0 +; CHECK-NEXT: retq +define <8 x float> @fmaddsubps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) { entry: br label %for.cond @@ -126,9 +517,11 @@ for.end: ret <8 x float> %c.addr.0 } -; CHECK-LABEL: fmsubaddps_loop: -; CHECK: vfmsubadd231ps %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}} -define <8 x float> @fmsubaddps_loop(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) { +; CHECK-LABEL: fmsubaddps_loop_256: +; CHECK: vfmsubadd231ps %ymm1, %ymm0, %ymm2 +; CHECK: vmovaps %ymm2, %ymm0 +; CHECK-NEXT: retq +define <8 x float> @fmsubaddps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) { entry: br label %for.cond @@ -150,9 +543,11 @@ for.end: ret <8 x float> %c.addr.0 } -; CHECK-LABEL: fmaddps_loop: -; CHECK: vfmadd231ps %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}} -define <8 x float> @fmaddps_loop(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) { +; CHECK-LABEL: fmaddps_loop_256: +; CHECK: vfmadd231ps %ymm1, %ymm0, %ymm2 +; CHECK: vmovaps %ymm2, %ymm0 +; CHECK-NEXT: retq +define <8 x float> @fmaddps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) { entry: br label %for.cond @@ -174,9 +569,11 @@ for.end: ret <8 x float> %c.addr.0 } -; CHECK-LABEL: fmsubps_loop: -; CHECK: vfmsub231ps %ymm{{[0-9]+}}, %ymm{{[0-9]+}}, %ymm{{[0-9]+}} -define <8 x float> @fmsubps_loop(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) { +; CHECK-LABEL: fmsubps_loop_256: +; CHECK: vfmsub231ps %ymm1, %ymm0, %ymm2 +; CHECK: vmovaps %ymm2, %ymm0 +; CHECK-NEXT: retq +define <8 x float> @fmsubps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) { entry: br label %for.cond @@ -198,7 +595,61 @@ for.end: ret <8 x float> %c.addr.0 } +; CHECK-LABEL: fnmaddps_loop_256: +; CHECK: vfnmadd231ps %ymm1, %ymm0, %ymm2 +; CHECK: vmovaps %ymm2, %ymm0 +; CHECK-NEXT: retq +define <8 x float> @fnmaddps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) { +entry: + br label %for.cond + +for.cond: + %c.addr.0 = phi <8 x float> [ %c, %entry ], [ %0, %for.inc ] + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ] + %cmp = icmp slt i32 %i.0, %iter + br i1 %cmp, label %for.body, label %for.end + +for.body: + br label %for.inc + +for.inc: + %0 = call <8 x float> @llvm.x86.fma.vfnmadd.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c.addr.0) + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: + ret <8 x float> %c.addr.0 +} + +; CHECK-LABEL: fnmsubps_loop_256: +; CHECK: vfnmsub231ps %ymm1, %ymm0, %ymm2 +; CHECK: vmovaps %ymm2, %ymm0 +; CHECK-NEXT: retq +define <8 x float> @fnmsubps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) { +entry: + br label %for.cond + +for.cond: + %c.addr.0 = phi <8 x float> [ %c, %entry ], [ %0, %for.inc ] + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ] + %cmp = icmp slt i32 %i.0, %iter + br i1 %cmp, label %for.body, label %for.end + +for.body: + br label %for.inc + +for.inc: + %0 = call <8 x float> @llvm.x86.fma.vfnmsub.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c.addr.0) + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: + ret <8 x float> %c.addr.0 +} + declare <8 x float> @llvm.x86.fma.vfmaddsub.ps.256(<8 x float>, <8 x float>, <8 x float>) declare <8 x float> @llvm.x86.fma.vfmsubadd.ps.256(<8 x float>, <8 x float>, <8 x float>) declare <8 x float> @llvm.x86.fma.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>) declare <8 x float> @llvm.x86.fma.vfmsub.ps.256(<8 x float>, <8 x float>, <8 x float>) +declare <8 x float> @llvm.x86.fma.vfnmadd.ps.256(<8 x float>, <8 x float>, <8 x float>) +declare <8 x float> @llvm.x86.fma.vfnmsub.ps.256(<8 x float>, <8 x float>, <8 x float>) diff --git a/test/CodeGen/X86/fma-intrinsics-x86.ll b/test/CodeGen/X86/fma-intrinsics-x86.ll index 881436386bac..cf4c8933fcab 100644 --- a/test/CodeGen/X86/fma-intrinsics-x86.ll +++ b/test/CodeGen/X86/fma-intrinsics-x86.ll @@ -1,95 +1,149 @@ ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -march=x86-64 -mcpu=corei7-avx -mattr=+fma | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -march=x86-64 -mcpu=core-avx2 -mattr=+fma,+avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA +; RUN: llc < %s -mtriple=x86_64-pc-windows -march=x86-64 -mcpu=core-avx2 -mattr=+fma,+avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA-WIN ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -march=x86-64 -mcpu=corei7-avx -mattr=+fma4 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA4 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=bdver2 -mattr=+avx,-fma | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA4 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=bdver2 -mattr=-fma4 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FMA ; VFMADD define <4 x float> @test_x86_fma_vfmadd_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmadd_ss: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmadd_ss: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps {{\(%rcx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vmovaps {{\(%rcx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vfmadd132ss (%rdx), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfmadd213ss %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmadd_ss: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmaddss %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmaddss %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.fma.vfmadd.ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) ret <4 x float> %res } + +define <4 x float> @test_x86_fma_vfmadd_bac_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 { +; CHECK-LABEL: test_x86_fma_vfmadd_bac_ss: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps {{\(%rdx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vmovaps {{\(%rdx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vfmadd132ss (%rcx), %xmm1, %xmm0 +; +; CHECK-FMA-NEXT: vfmadd213ss %xmm2, %xmm0, %xmm1 +; CHECK-FMA-NEXT: vmovaps %xmm1, %xmm0 +; +; CHECK-FMA4-NEXT: vfmaddss %xmm2, %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfmadd.ss(<4 x float> %a1, <4 x float> %a0, <4 x float> %a2) + ret <4 x float> %res +} declare <4 x float> @llvm.x86.fma.vfmadd.ss(<4 x float>, <4 x float>, <4 x float>) define <2 x double> @test_x86_fma_vfmadd_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmadd_sd: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmadd_sd: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd {{\(%rcx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vmovapd {{\(%rcx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vfmadd132sd (%rdx), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfmadd213sd %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmadd_sd: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmaddsd %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmaddsd %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.fma.vfmadd.sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) ret <2 x double> %res } + +define <2 x double> @test_x86_fma_vfmadd_bac_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 { +; CHECK-LABEL: test_x86_fma_vfmadd_bac_sd: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd {{\(%rdx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vmovapd {{\(%rdx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vfmadd132sd (%rcx), %xmm1, %xmm0 +; +; CHECK-FMA-NEXT: vfmadd213sd %xmm2, %xmm0, %xmm1 +; CHECK-FMA-NEXT: vmovapd %xmm1, %xmm0 +; +; CHECK-FMA4-NEXT: vfmaddsd %xmm2, %xmm0, %xmm1, %xmm0 +; +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfmadd.sd(<2 x double> %a1, <2 x double> %a0, <2 x double> %a2) + ret <2 x double> %res +} declare <2 x double> @llvm.x86.fma.vfmadd.sd(<2 x double>, <2 x double>, <2 x double>) define <4 x float> @test_x86_fma_vfmadd_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmadd_ps: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmadd_ps: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfmadd213ps (%r8), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfmadd213ps %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmadd_ps: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmaddps %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmaddps %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.fma.vfmadd.ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) ret <4 x float> %res } declare <4 x float> @llvm.x86.fma.vfmadd.ps(<4 x float>, <4 x float>, <4 x float>) define <2 x double> @test_x86_fma_vfmadd_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmadd_pd: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmadd_pd: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfmadd213pd (%r8), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfmadd213pd %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmadd_pd: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmaddpd %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmaddpd %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.fma.vfmadd.pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) ret <2 x double> %res } declare <2 x double> @llvm.x86.fma.vfmadd.pd(<2 x double>, <2 x double>, <2 x double>) define <8 x float> @test_x86_fma_vfmadd_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmadd_ps_256: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmadd_ps_256: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfmadd213ps (%r8), %ymm1, %ymm0 +; ; CHECK-FMA-NEXT: vfmadd213ps %ymm2, %ymm1, %ymm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmadd_ps_256: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmaddps %ymm2, %ymm1, %ymm0, %ymm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmaddps %ymm2, %ymm1, %ymm0, %ymm0 +; +; CHECK-NEXT: retq %res = call <8 x float> @llvm.x86.fma.vfmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) ret <8 x float> %res } declare <8 x float> @llvm.x86.fma.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>) define <4 x double> @test_x86_fma_vfmadd_pd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmadd_pd_256: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmadd_pd_256: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfmadd213pd (%r8), %ymm1, %ymm0 +; ; CHECK-FMA-NEXT: vfmadd213pd %ymm2, %ymm1, %ymm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmadd_pd_256: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmaddpd %ymm2, %ymm1, %ymm0, %ymm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmaddpd %ymm2, %ymm1, %ymm0, %ymm0 +; +; CHECK-NEXT: retq %res = call <4 x double> @llvm.x86.fma.vfmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) ret <4 x double> %res } @@ -97,90 +151,144 @@ declare <4 x double> @llvm.x86.fma.vfmadd.pd.256(<4 x double>, <4 x double>, <4 ; VFMSUB define <4 x float> @test_x86_fma_vfmsub_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmsub_ss: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmsub_ss: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps {{\(%rcx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vmovaps {{\(%rcx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vfmsub132ss (%rdx), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfmsub213ss %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmsub_ss: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmsubss %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmsubss %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.fma.vfmsub.ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) ret <4 x float> %res } + +define <4 x float> @test_x86_fma_vfmsub_bac_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 { +; CHECK-LABEL: test_x86_fma_vfmsub_bac_ss: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps {{\(%rdx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vmovaps {{\(%rdx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vfmsub132ss (%rcx), %xmm1, %xmm0 +; +; CHECK-FMA-NEXT: vfmsub213ss %xmm2, %xmm0, %xmm1 +; CHECK-FMA-NEXT: vmovaps %xmm1, %xmm0 +; +; CHECK-FMA4-NEXT: vfmsubss %xmm2, %xmm0, %xmm1, %xmm0 +; +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfmsub.ss(<4 x float> %a1, <4 x float> %a0, <4 x float> %a2) + ret <4 x float> %res +} declare <4 x float> @llvm.x86.fma.vfmsub.ss(<4 x float>, <4 x float>, <4 x float>) define <2 x double> @test_x86_fma_vfmsub_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmsub_sd: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmsub_sd: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd {{\(%rcx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vmovapd {{\(%rcx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vfmsub132sd (%rdx), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfmsub213sd %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmsub_sd: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmsubsd %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmsubsd %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.fma.vfmsub.sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) ret <2 x double> %res } + +define <2 x double> @test_x86_fma_vfmsub_bac_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 { +; CHECK-LABEL: test_x86_fma_vfmsub_bac_sd: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd {{\(%rdx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vmovapd {{\(%rdx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vfmsub132sd (%rcx), %xmm1, %xmm0 +; +; CHECK-FMA-NEXT: vfmsub213sd %xmm2, %xmm0, %xmm1 +; CHECK-FMA-NEXT: vmovapd %xmm1, %xmm0 +; +; CHECK-FMA4-NEXT: vfmsubsd %xmm2, %xmm0, %xmm1, %xmm0 +; +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfmsub.sd(<2 x double> %a1, <2 x double> %a0, <2 x double> %a2) + ret <2 x double> %res +} declare <2 x double> @llvm.x86.fma.vfmsub.sd(<2 x double>, <2 x double>, <2 x double>) define <4 x float> @test_x86_fma_vfmsub_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmsub_ps: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmsub_ps: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfmsub213ps (%r8), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfmsub213ps %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmsub_ps: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmsubps %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmsubps %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.fma.vfmsub.ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) ret <4 x float> %res } declare <4 x float> @llvm.x86.fma.vfmsub.ps(<4 x float>, <4 x float>, <4 x float>) define <2 x double> @test_x86_fma_vfmsub_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmsub_pd: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmsub_pd: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfmsub213pd (%r8), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfmsub213pd %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmsub_pd: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmsubpd %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmsubpd %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.fma.vfmsub.pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) ret <2 x double> %res } declare <2 x double> @llvm.x86.fma.vfmsub.pd(<2 x double>, <2 x double>, <2 x double>) define <8 x float> @test_x86_fma_vfmsub_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmsub_ps_256: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmsub_ps_256: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfmsub213ps (%r8), %ymm1, %ymm0 +; ; CHECK-FMA-NEXT: vfmsub213ps %ymm2, %ymm1, %ymm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmsub_ps_256: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmsubps %ymm2, %ymm1, %ymm0, %ymm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmsubps %ymm2, %ymm1, %ymm0, %ymm0 +; +; CHECK-NEXT: retq %res = call <8 x float> @llvm.x86.fma.vfmsub.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) ret <8 x float> %res } declare <8 x float> @llvm.x86.fma.vfmsub.ps.256(<8 x float>, <8 x float>, <8 x float>) define <4 x double> @test_x86_fma_vfmsub_pd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmsub_pd_256: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmsub_pd_256: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfmsub213pd (%r8), %ymm1, %ymm0 +; ; CHECK-FMA-NEXT: vfmsub213pd %ymm2, %ymm1, %ymm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmsub_pd_256: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmsubpd %ymm2, %ymm1, %ymm0, %ymm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmsubpd %ymm2, %ymm1, %ymm0, %ymm0 +; +; CHECK-NEXT: retq %res = call <4 x double> @llvm.x86.fma.vfmsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) ret <4 x double> %res } @@ -188,90 +296,144 @@ declare <4 x double> @llvm.x86.fma.vfmsub.pd.256(<4 x double>, <4 x double>, <4 ; VFNMADD define <4 x float> @test_x86_fma_vfnmadd_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfnmadd_ss: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfnmadd_ss: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps {{\(%rcx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vmovaps {{\(%rcx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vfnmadd132ss (%rdx), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfnmadd213ss %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfnmadd_ss: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfnmaddss %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfnmaddss %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.fma.vfnmadd.ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) ret <4 x float> %res } + +define <4 x float> @test_x86_fma_vfnmadd_bac_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 { +; CHECK-LABEL: test_x86_fma_vfnmadd_bac_ss: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps {{\(%rdx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vmovaps {{\(%rdx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vfnmadd132ss (%rcx), %xmm1, %xmm0 +; +; CHECK-FMA-NEXT: vfnmadd213ss %xmm2, %xmm0, %xmm1 +; CHECK-FMA-NEXT: vmovaps %xmm1, %xmm0 +; +; CHECK-FMA4-NEXT: vfnmaddss %xmm2, %xmm0, %xmm1, %xmm0 +; +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfnmadd.ss(<4 x float> %a1, <4 x float> %a0, <4 x float> %a2) + ret <4 x float> %res +} declare <4 x float> @llvm.x86.fma.vfnmadd.ss(<4 x float>, <4 x float>, <4 x float>) define <2 x double> @test_x86_fma_vfnmadd_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfnmadd_sd: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfnmadd_sd: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd {{\(%rcx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vmovapd {{\(%rcx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vfnmadd132sd (%rdx), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfnmadd213sd %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfnmadd_sd: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfnmaddsd %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfnmaddsd %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.fma.vfnmadd.sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) ret <2 x double> %res } + +define <2 x double> @test_x86_fma_vfnmadd_bac_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 { +; CHECK-LABEL: test_x86_fma_vfnmadd_bac_sd: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd {{\(%rdx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vmovapd {{\(%rdx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vfnmadd132sd (%rcx), %xmm1, %xmm0 +; +; CHECK-FMA-NEXT: vfnmadd213sd %xmm2, %xmm0, %xmm1 +; CHECK-FMA-NEXT: vmovapd %xmm1, %xmm0 +; +; CHECK-FMA4-NEXT: vfnmaddsd %xmm2, %xmm0, %xmm1, %xmm0 +; +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfnmadd.sd(<2 x double> %a1, <2 x double> %a0, <2 x double> %a2) + ret <2 x double> %res +} declare <2 x double> @llvm.x86.fma.vfnmadd.sd(<2 x double>, <2 x double>, <2 x double>) define <4 x float> @test_x86_fma_vfnmadd_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfnmadd_ps: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfnmadd_ps: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfnmadd213ps (%r8), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfnmadd213ps %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfnmadd_ps: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfnmaddps %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfnmaddps %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.fma.vfnmadd.ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) ret <4 x float> %res } declare <4 x float> @llvm.x86.fma.vfnmadd.ps(<4 x float>, <4 x float>, <4 x float>) define <2 x double> @test_x86_fma_vfnmadd_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfnmadd_pd: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfnmadd_pd: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfnmadd213pd (%r8), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfnmadd213pd %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfnmadd_pd: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfnmaddpd %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfnmaddpd %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.fma.vfnmadd.pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) ret <2 x double> %res } declare <2 x double> @llvm.x86.fma.vfnmadd.pd(<2 x double>, <2 x double>, <2 x double>) define <8 x float> @test_x86_fma_vfnmadd_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfnmadd_ps_256: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfnmadd_ps_256: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfnmadd213ps (%r8), %ymm1, %ymm0 +; ; CHECK-FMA-NEXT: vfnmadd213ps %ymm2, %ymm1, %ymm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfnmadd_ps_256: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfnmaddps %ymm2, %ymm1, %ymm0, %ymm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfnmaddps %ymm2, %ymm1, %ymm0, %ymm0 +; +; CHECK-NEXT: retq %res = call <8 x float> @llvm.x86.fma.vfnmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) ret <8 x float> %res } declare <8 x float> @llvm.x86.fma.vfnmadd.ps.256(<8 x float>, <8 x float>, <8 x float>) define <4 x double> @test_x86_fma_vfnmadd_pd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfnmadd_pd_256: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfnmadd_pd_256: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfnmadd213pd (%r8), %ymm1, %ymm0 +; ; CHECK-FMA-NEXT: vfnmadd213pd %ymm2, %ymm1, %ymm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfnmadd_pd_256: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfnmaddpd %ymm2, %ymm1, %ymm0, %ymm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfnmaddpd %ymm2, %ymm1, %ymm0, %ymm0 +; +; CHECK-NEXT: retq %res = call <4 x double> @llvm.x86.fma.vfnmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) ret <4 x double> %res } @@ -279,90 +441,144 @@ declare <4 x double> @llvm.x86.fma.vfnmadd.pd.256(<4 x double>, <4 x double>, <4 ; VFNMSUB define <4 x float> @test_x86_fma_vfnmsub_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfnmsub_ss: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfnmsub_ss: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps {{\(%rcx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vmovaps {{\(%rcx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vfnmsub132ss (%rdx), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfnmsub213ss %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfnmsub_ss: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfnmsubss %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfnmsubss %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.fma.vfnmsub.ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) ret <4 x float> %res } + +define <4 x float> @test_x86_fma_vfnmsub_bac_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 { +; CHECK-LABEL: test_x86_fma_vfnmsub_bac_ss: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps {{\(%rdx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vmovaps {{\(%rdx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vfnmsub132ss (%rcx), %xmm1, %xmm0 +; +; CHECK-FMA-NEXT: vfnmsub213ss %xmm2, %xmm0, %xmm1 +; CHECK-FMA-NEXT: vmovaps %xmm1, %xmm0 +; +; CHECK-FMA4-NEXT: vfnmsubss %xmm2, %xmm0, %xmm1, %xmm0 +; +; CHECK-NEXT: retq + %res = call <4 x float> @llvm.x86.fma.vfnmsub.ss(<4 x float> %a1, <4 x float> %a0, <4 x float> %a2) + ret <4 x float> %res +} declare <4 x float> @llvm.x86.fma.vfnmsub.ss(<4 x float>, <4 x float>, <4 x float>) define <2 x double> @test_x86_fma_vfnmsub_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfnmsub_sd: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfnmsub_sd: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd {{\(%rcx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vmovapd {{\(%rcx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vfnmsub132sd (%rdx), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfnmsub213sd %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfnmsub_sd: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfnmsubsd %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfnmsubsd %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.fma.vfnmsub.sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) ret <2 x double> %res } + +define <2 x double> @test_x86_fma_vfnmsub_bac_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 { +; CHECK-LABEL: test_x86_fma_vfnmsub_bac_sd: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd {{\(%rdx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vmovapd {{\(%rdx\), %xmm0|\(%r8\), %xmm1}} +; CHECK-FMA-WIN-NEXT: vfnmsub132sd (%rcx), %xmm1, %xmm0 +; +; CHECK-FMA-NEXT: vfnmsub213sd %xmm2, %xmm0, %xmm1 +; CHECK-FMA-NEXT: vmovapd %xmm1, %xmm0 +; +; CHECK-FMA4-NEXT: vfnmsubsd %xmm2, %xmm0, %xmm1, %xmm0 +; +; CHECK-NEXT: retq + %res = call <2 x double> @llvm.x86.fma.vfnmsub.sd(<2 x double> %a1, <2 x double> %a0, <2 x double> %a2) + ret <2 x double> %res +} declare <2 x double> @llvm.x86.fma.vfnmsub.sd(<2 x double>, <2 x double>, <2 x double>) define <4 x float> @test_x86_fma_vfnmsub_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfnmsub_ps: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfnmsub_ps: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfnmsub213ps (%r8), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfnmsub213ps %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfnmsub_ps: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfnmsubps %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfnmsubps %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.fma.vfnmsub.ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) ret <4 x float> %res } declare <4 x float> @llvm.x86.fma.vfnmsub.ps(<4 x float>, <4 x float>, <4 x float>) define <2 x double> @test_x86_fma_vfnmsub_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfnmsub_pd: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfnmsub_pd: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfnmsub213pd (%r8), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfnmsub213pd %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfnmsub_pd: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfnmsubpd %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfnmsubpd %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.fma.vfnmsub.pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) ret <2 x double> %res } declare <2 x double> @llvm.x86.fma.vfnmsub.pd(<2 x double>, <2 x double>, <2 x double>) define <8 x float> @test_x86_fma_vfnmsub_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfnmsub_ps_256: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfnmsub_ps_256: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfnmsub213ps (%r8), %ymm1, %ymm0 +; ; CHECK-FMA-NEXT: vfnmsub213ps %ymm2, %ymm1, %ymm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfnmsub_ps_256: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfnmsubps %ymm2, %ymm1, %ymm0, %ymm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfnmsubps %ymm2, %ymm1, %ymm0, %ymm0 +; +; CHECK-NEXT: retq %res = call <8 x float> @llvm.x86.fma.vfnmsub.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) ret <8 x float> %res } declare <8 x float> @llvm.x86.fma.vfnmsub.ps.256(<8 x float>, <8 x float>, <8 x float>) define <4 x double> @test_x86_fma_vfnmsub_pd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfnmsub_pd_256: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfnmsub_pd_256: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfnmsub213pd (%r8), %ymm1, %ymm0 +; ; CHECK-FMA-NEXT: vfnmsub213pd %ymm2, %ymm1, %ymm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfnmsub_pd_256: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfnmsubpd %ymm2, %ymm1, %ymm0, %ymm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfnmsubpd %ymm2, %ymm1, %ymm0, %ymm0 +; +; CHECK-NEXT: retq %res = call <4 x double> @llvm.x86.fma.vfnmsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) ret <4 x double> %res } @@ -370,60 +586,72 @@ declare <4 x double> @llvm.x86.fma.vfnmsub.pd.256(<4 x double>, <4 x double>, <4 ; VFMADDSUB define <4 x float> @test_x86_fma_vfmaddsub_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmaddsub_ps: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmaddsub_ps: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfmaddsub213ps (%r8), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfmaddsub213ps %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmaddsub_ps: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmaddsubps %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmaddsubps %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.fma.vfmaddsub.ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) ret <4 x float> %res } declare <4 x float> @llvm.x86.fma.vfmaddsub.ps(<4 x float>, <4 x float>, <4 x float>) define <2 x double> @test_x86_fma_vfmaddsub_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmaddsub_pd: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmaddsub_pd: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfmaddsub213pd (%r8), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfmaddsub213pd %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmaddsub_pd: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmaddsubpd %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmaddsubpd %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.fma.vfmaddsub.pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) ret <2 x double> %res } declare <2 x double> @llvm.x86.fma.vfmaddsub.pd(<2 x double>, <2 x double>, <2 x double>) define <8 x float> @test_x86_fma_vfmaddsub_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmaddsub_ps_256: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmaddsub_ps_256: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfmaddsub213ps (%r8), %ymm1, %ymm0 +; ; CHECK-FMA-NEXT: vfmaddsub213ps %ymm2, %ymm1, %ymm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmaddsub_ps_256: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmaddsubps %ymm2, %ymm1, %ymm0, %ymm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmaddsubps %ymm2, %ymm1, %ymm0, %ymm0 +; +; CHECK-NEXT: retq %res = call <8 x float> @llvm.x86.fma.vfmaddsub.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) ret <8 x float> %res } declare <8 x float> @llvm.x86.fma.vfmaddsub.ps.256(<8 x float>, <8 x float>, <8 x float>) define <4 x double> @test_x86_fma_vfmaddsub_pd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmaddsub_pd_256: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmaddsub_pd_256: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfmaddsub213pd (%r8), %ymm1, %ymm0 +; ; CHECK-FMA-NEXT: vfmaddsub213pd %ymm2, %ymm1, %ymm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmaddsub_pd_256: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmaddsubpd %ymm2, %ymm1, %ymm0, %ymm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmaddsubpd %ymm2, %ymm1, %ymm0, %ymm0 +; +; CHECK-NEXT: retq %res = call <4 x double> @llvm.x86.fma.vfmaddsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) ret <4 x double> %res } @@ -431,60 +659,72 @@ declare <4 x double> @llvm.x86.fma.vfmaddsub.pd.256(<4 x double>, <4 x double>, ; VFMSUBADD define <4 x float> @test_x86_fma_vfmsubadd_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmsubadd_ps: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmsubadd_ps: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfmsubadd213ps (%r8), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfmsubadd213ps %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmsubadd_ps: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmsubaddps %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmsubaddps %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <4 x float> @llvm.x86.fma.vfmsubadd.ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) ret <4 x float> %res } declare <4 x float> @llvm.x86.fma.vfmsubadd.ps(<4 x float>, <4 x float>, <4 x float>) define <2 x double> @test_x86_fma_vfmsubadd_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmsubadd_pd: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmsubadd_pd: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %xmm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfmsubadd213pd (%r8), %xmm1, %xmm0 +; ; CHECK-FMA-NEXT: vfmsubadd213pd %xmm2, %xmm1, %xmm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmsubadd_pd: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmsubaddpd %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmsubaddpd %xmm2, %xmm1, %xmm0, %xmm0 +; +; CHECK-NEXT: retq %res = call <2 x double> @llvm.x86.fma.vfmsubadd.pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) ret <2 x double> %res } declare <2 x double> @llvm.x86.fma.vfmsubadd.pd(<2 x double>, <2 x double>, <2 x double>) define <8 x float> @test_x86_fma_vfmsubadd_ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmsubadd_ps_256: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmsubadd_ps_256: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovaps (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfmsubadd213ps (%r8), %ymm1, %ymm0 +; ; CHECK-FMA-NEXT: vfmsubadd213ps %ymm2, %ymm1, %ymm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmsubadd_ps_256: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmsubaddps %ymm2, %ymm1, %ymm0, %ymm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmsubaddps %ymm2, %ymm1, %ymm0, %ymm0 +; +; CHECK-NEXT: retq %res = call <8 x float> @llvm.x86.fma.vfmsubadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) ret <8 x float> %res } declare <8 x float> @llvm.x86.fma.vfmsubadd.ps.256(<8 x float>, <8 x float>, <8 x float>) define <4 x double> @test_x86_fma_vfmsubadd_pd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 { -; CHECK-FMA-LABEL: test_x86_fma_vfmsubadd_pd_256: -; CHECK-FMA: # BB#0: +; CHECK-LABEL: test_x86_fma_vfmsubadd_pd_256: +; CHECK-NEXT: # BB#0: +; +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vmovapd (%{{(rcx|rdx)}}), %ymm{{0|1}} +; CHECK-FMA-WIN-NEXT: vfmsubadd213pd (%r8), %ymm1, %ymm0 +; ; CHECK-FMA-NEXT: vfmsubadd213pd %ymm2, %ymm1, %ymm0 -; CHECK-FMA-NEXT: retq ; -; CHECK-FMA4-LABEL: test_x86_fma_vfmsubadd_pd_256: -; CHECK-FMA4: # BB#0: -; CHECK-FMA4-NEXT: vfmsubaddpd %ymm2, %ymm1, %ymm0, %ymm0 -; CHECK-FMA4-NEXT: retq +; CHECK-FMA4-NEXT: vfmsubaddpd %ymm2, %ymm1, %ymm0, %ymm0 +; +; CHECK-NEXT: retq %res = call <4 x double> @llvm.x86.fma.vfmsubadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) ret <4 x double> %res } diff --git a/test/CodeGen/X86/fma-scalar-memfold.ll b/test/CodeGen/X86/fma-scalar-memfold.ll new file mode 100644 index 000000000000..0ceaa562a5d4 --- /dev/null +++ b/test/CodeGen/X86/fma-scalar-memfold.ll @@ -0,0 +1,383 @@ +; RUN: llc < %s -mtriple=x86_64-pc-win32 -mcpu=core-avx2 | FileCheck %s + +attributes #0 = { nounwind } + +declare <4 x float> @llvm.x86.fma.vfmadd.ss(<4 x float>, <4 x float>, <4 x float>) +declare <4 x float> @llvm.x86.fma.vfmsub.ss(<4 x float>, <4 x float>, <4 x float>) +declare <4 x float> @llvm.x86.fma.vfnmadd.ss(<4 x float>, <4 x float>, <4 x float>) +declare <4 x float> @llvm.x86.fma.vfnmsub.ss(<4 x float>, <4 x float>, <4 x float>) + +declare <2 x double> @llvm.x86.fma.vfmadd.sd(<2 x double>, <2 x double>, <2 x double>) +declare <2 x double> @llvm.x86.fma.vfmsub.sd(<2 x double>, <2 x double>, <2 x double>) +declare <2 x double> @llvm.x86.fma.vfnmadd.sd(<2 x double>, <2 x double>, <2 x double>) +declare <2 x double> @llvm.x86.fma.vfnmsub.sd(<2 x double>, <2 x double>, <2 x double>) + +define void @fmadd_aab_ss(float* %a, float* %b) #0 { +; CHECK-LABEL: fmadd_aab_ss: +; CHECK: vmovss (%rcx), %[[XMM:xmm[0-9]+]] +; CHECK-NEXT: vfmadd213ss (%rdx), %[[XMM]], %[[XMM]] +; CHECK-NEXT: vmovss %[[XMM]], (%rcx) +; CHECK-NEXT: ret + %a.val = load float, float* %a + %av0 = insertelement <4 x float> undef, float %a.val, i32 0 + %av1 = insertelement <4 x float> %av0, float 0.000000e+00, i32 1 + %av2 = insertelement <4 x float> %av1, float 0.000000e+00, i32 2 + %av = insertelement <4 x float> %av2, float 0.000000e+00, i32 3 + + %b.val = load float, float* %b + %bv0 = insertelement <4 x float> undef, float %b.val, i32 0 + %bv1 = insertelement <4 x float> %bv0, float 0.000000e+00, i32 1 + %bv2 = insertelement <4 x float> %bv1, float 0.000000e+00, i32 2 + %bv = insertelement <4 x float> %bv2, float 0.000000e+00, i32 3 + + %vr = call <4 x float> @llvm.x86.fma.vfmadd.ss(<4 x float> %av, <4 x float> %av, <4 x float> %bv) + + %sr = extractelement <4 x float> %vr, i32 0 + store float %sr, float* %a + ret void +} + +define void @fmadd_aba_ss(float* %a, float* %b) #0 { +; CHECK-LABEL: fmadd_aba_ss: +; CHECK: vmovss (%rcx), %[[XMM:xmm[0-9]+]] +; CHECK-NEXT: vfmadd132ss (%rdx), %[[XMM]], %[[XMM]] +; CHECK-NEXT: vmovss %[[XMM]], (%rcx) +; CHECK-NEXT: ret + %a.val = load float, float* %a + %av0 = insertelement <4 x float> undef, float %a.val, i32 0 + %av1 = insertelement <4 x float> %av0, float 0.000000e+00, i32 1 + %av2 = insertelement <4 x float> %av1, float 0.000000e+00, i32 2 + %av = insertelement <4 x float> %av2, float 0.000000e+00, i32 3 + + %b.val = load float, float* %b + %bv0 = insertelement <4 x float> undef, float %b.val, i32 0 + %bv1 = insertelement <4 x float> %bv0, float 0.000000e+00, i32 1 + %bv2 = insertelement <4 x float> %bv1, float 0.000000e+00, i32 2 + %bv = insertelement <4 x float> %bv2, float 0.000000e+00, i32 3 + + %vr = call <4 x float> @llvm.x86.fma.vfmadd.ss(<4 x float> %av, <4 x float> %bv, <4 x float> %av) + + %sr = extractelement <4 x float> %vr, i32 0 + store float %sr, float* %a + ret void +} + +define void @fmsub_aab_ss(float* %a, float* %b) #0 { +; CHECK-LABEL: fmsub_aab_ss: +; CHECK: vmovss (%rcx), %[[XMM:xmm[0-9]+]] +; CHECK-NEXT: vfmsub213ss (%rdx), %[[XMM]], %[[XMM]] +; CHECK-NEXT: vmovss %[[XMM]], (%rcx) +; CHECK-NEXT: ret + %a.val = load float, float* %a + %av0 = insertelement <4 x float> undef, float %a.val, i32 0 + %av1 = insertelement <4 x float> %av0, float 0.000000e+00, i32 1 + %av2 = insertelement <4 x float> %av1, float 0.000000e+00, i32 2 + %av = insertelement <4 x float> %av2, float 0.000000e+00, i32 3 + + %b.val = load float, float* %b + %bv0 = insertelement <4 x float> undef, float %b.val, i32 0 + %bv1 = insertelement <4 x float> %bv0, float 0.000000e+00, i32 1 + %bv2 = insertelement <4 x float> %bv1, float 0.000000e+00, i32 2 + %bv = insertelement <4 x float> %bv2, float 0.000000e+00, i32 3 + + %vr = call <4 x float> @llvm.x86.fma.vfmsub.ss(<4 x float> %av, <4 x float> %av, <4 x float> %bv) + + %sr = extractelement <4 x float> %vr, i32 0 + store float %sr, float* %a + ret void +} + +define void @fmsub_aba_ss(float* %a, float* %b) #0 { +; CHECK-LABEL: fmsub_aba_ss: +; CHECK: vmovss (%rcx), %[[XMM:xmm[0-9]+]] +; CHECK-NEXT: vfmsub132ss (%rdx), %[[XMM]], %[[XMM]] +; CHECK-NEXT: vmovss %[[XMM]], (%rcx) +; CHECK-NEXT: ret + %a.val = load float, float* %a + %av0 = insertelement <4 x float> undef, float %a.val, i32 0 + %av1 = insertelement <4 x float> %av0, float 0.000000e+00, i32 1 + %av2 = insertelement <4 x float> %av1, float 0.000000e+00, i32 2 + %av = insertelement <4 x float> %av2, float 0.000000e+00, i32 3 + + %b.val = load float, float* %b + %bv0 = insertelement <4 x float> undef, float %b.val, i32 0 + %bv1 = insertelement <4 x float> %bv0, float 0.000000e+00, i32 1 + %bv2 = insertelement <4 x float> %bv1, float 0.000000e+00, i32 2 + %bv = insertelement <4 x float> %bv2, float 0.000000e+00, i32 3 + + %vr = call <4 x float> @llvm.x86.fma.vfmsub.ss(<4 x float> %av, <4 x float> %bv, <4 x float> %av) + + %sr = extractelement <4 x float> %vr, i32 0 + store float %sr, float* %a + ret void +} + +define void @fnmadd_aab_ss(float* %a, float* %b) #0 { +; CHECK-LABEL: fnmadd_aab_ss: +; CHECK: vmovss (%rcx), %[[XMM:xmm[0-9]+]] +; CHECK-NEXT: vfnmadd213ss (%rdx), %[[XMM]], %[[XMM]] +; CHECK-NEXT: vmovss %[[XMM]], (%rcx) +; CHECK-NEXT: ret + %a.val = load float, float* %a + %av0 = insertelement <4 x float> undef, float %a.val, i32 0 + %av1 = insertelement <4 x float> %av0, float 0.000000e+00, i32 1 + %av2 = insertelement <4 x float> %av1, float 0.000000e+00, i32 2 + %av = insertelement <4 x float> %av2, float 0.000000e+00, i32 3 + + %b.val = load float, float* %b + %bv0 = insertelement <4 x float> undef, float %b.val, i32 0 + %bv1 = insertelement <4 x float> %bv0, float 0.000000e+00, i32 1 + %bv2 = insertelement <4 x float> %bv1, float 0.000000e+00, i32 2 + %bv = insertelement <4 x float> %bv2, float 0.000000e+00, i32 3 + + %vr = call <4 x float> @llvm.x86.fma.vfnmadd.ss(<4 x float> %av, <4 x float> %av, <4 x float> %bv) + + %sr = extractelement <4 x float> %vr, i32 0 + store float %sr, float* %a + ret void +} + +define void @fnmadd_aba_ss(float* %a, float* %b) #0 { +; CHECK-LABEL: fnmadd_aba_ss: +; CHECK: vmovss (%rcx), %[[XMM:xmm[0-9]+]] +; CHECK-NEXT: vfnmadd132ss (%rdx), %[[XMM]], %[[XMM]] +; CHECK-NEXT: vmovss %[[XMM]], (%rcx) +; CHECK-NEXT: ret + %a.val = load float, float* %a + %av0 = insertelement <4 x float> undef, float %a.val, i32 0 + %av1 = insertelement <4 x float> %av0, float 0.000000e+00, i32 1 + %av2 = insertelement <4 x float> %av1, float 0.000000e+00, i32 2 + %av = insertelement <4 x float> %av2, float 0.000000e+00, i32 3 + + %b.val = load float, float* %b + %bv0 = insertelement <4 x float> undef, float %b.val, i32 0 + %bv1 = insertelement <4 x float> %bv0, float 0.000000e+00, i32 1 + %bv2 = insertelement <4 x float> %bv1, float 0.000000e+00, i32 2 + %bv = insertelement <4 x float> %bv2, float 0.000000e+00, i32 3 + + %vr = call <4 x float> @llvm.x86.fma.vfnmadd.ss(<4 x float> %av, <4 x float> %bv, <4 x float> %av) + + %sr = extractelement <4 x float> %vr, i32 0 + store float %sr, float* %a + ret void +} + +define void @fnmsub_aab_ss(float* %a, float* %b) #0 { +; CHECK-LABEL: fnmsub_aab_ss: +; CHECK: vmovss (%rcx), %[[XMM:xmm[0-9]+]] +; CHECK-NEXT: vfnmsub213ss (%rdx), %[[XMM]], %[[XMM]] +; CHECK-NEXT: vmovss %[[XMM]], (%rcx) +; CHECK-NEXT: ret + %a.val = load float, float* %a + %av0 = insertelement <4 x float> undef, float %a.val, i32 0 + %av1 = insertelement <4 x float> %av0, float 0.000000e+00, i32 1 + %av2 = insertelement <4 x float> %av1, float 0.000000e+00, i32 2 + %av = insertelement <4 x float> %av2, float 0.000000e+00, i32 3 + + %b.val = load float, float* %b + %bv0 = insertelement <4 x float> undef, float %b.val, i32 0 + %bv1 = insertelement <4 x float> %bv0, float 0.000000e+00, i32 1 + %bv2 = insertelement <4 x float> %bv1, float 0.000000e+00, i32 2 + %bv = insertelement <4 x float> %bv2, float 0.000000e+00, i32 3 + + %vr = call <4 x float> @llvm.x86.fma.vfnmsub.ss(<4 x float> %av, <4 x float> %av, <4 x float> %bv) + + %sr = extractelement <4 x float> %vr, i32 0 + store float %sr, float* %a + ret void +} + +define void @fnmsub_aba_ss(float* %a, float* %b) #0 { +; CHECK-LABEL: fnmsub_aba_ss: +; CHECK: vmovss (%rcx), %[[XMM:xmm[0-9]+]] +; CHECK-NEXT: vfnmsub132ss (%rdx), %[[XMM]], %[[XMM]] +; CHECK-NEXT: vmovss %[[XMM]], (%rcx) +; CHECK-NEXT: ret + %a.val = load float, float* %a + %av0 = insertelement <4 x float> undef, float %a.val, i32 0 + %av1 = insertelement <4 x float> %av0, float 0.000000e+00, i32 1 + %av2 = insertelement <4 x float> %av1, float 0.000000e+00, i32 2 + %av = insertelement <4 x float> %av2, float 0.000000e+00, i32 3 + + %b.val = load float, float* %b + %bv0 = insertelement <4 x float> undef, float %b.val, i32 0 + %bv1 = insertelement <4 x float> %bv0, float 0.000000e+00, i32 1 + %bv2 = insertelement <4 x float> %bv1, float 0.000000e+00, i32 2 + %bv = insertelement <4 x float> %bv2, float 0.000000e+00, i32 3 + + %vr = call <4 x float> @llvm.x86.fma.vfnmsub.ss(<4 x float> %av, <4 x float> %bv, <4 x float> %av) + + %sr = extractelement <4 x float> %vr, i32 0 + store float %sr, float* %a + ret void +} + +define void @fmadd_aab_sd(double* %a, double* %b) #0 { +; CHECK-LABEL: fmadd_aab_sd: +; CHECK: vmovsd (%rcx), %[[XMM:xmm[0-9]+]] +; CHECK-NEXT: vfmadd213sd (%rdx), %[[XMM]], %[[XMM]] +; CHECK-NEXT: vmovlpd %[[XMM]], (%rcx) +; CHECK-NEXT: ret + %a.val = load double, double* %a + %av0 = insertelement <2 x double> undef, double %a.val, i32 0 + %av = insertelement <2 x double> %av0, double 0.000000e+00, i32 1 + + %b.val = load double, double* %b + %bv0 = insertelement <2 x double> undef, double %b.val, i32 0 + %bv = insertelement <2 x double> %bv0, double 0.000000e+00, i32 1 + + %vr = call <2 x double> @llvm.x86.fma.vfmadd.sd(<2 x double> %av, <2 x double> %av, <2 x double> %bv) + + %sr = extractelement <2 x double> %vr, i32 0 + store double %sr, double* %a + ret void +} + +define void @fmadd_aba_sd(double* %a, double* %b) #0 { +; CHECK-LABEL: fmadd_aba_sd: +; CHECK: vmovsd (%rcx), %[[XMM:xmm[0-9]+]] +; CHECK-NEXT: vfmadd132sd (%rdx), %[[XMM]], %[[XMM]] +; CHECK-NEXT: vmovlpd %[[XMM]], (%rcx) +; CHECK-NEXT: ret + %a.val = load double, double* %a + %av0 = insertelement <2 x double> undef, double %a.val, i32 0 + %av = insertelement <2 x double> %av0, double 0.000000e+00, i32 1 + + %b.val = load double, double* %b + %bv0 = insertelement <2 x double> undef, double %b.val, i32 0 + %bv = insertelement <2 x double> %bv0, double 0.000000e+00, i32 1 + + %vr = call <2 x double> @llvm.x86.fma.vfmadd.sd(<2 x double> %av, <2 x double> %bv, <2 x double> %av) + + %sr = extractelement <2 x double> %vr, i32 0 + store double %sr, double* %a + ret void +} + +define void @fmsub_aab_sd(double* %a, double* %b) #0 { +; CHECK-LABEL: fmsub_aab_sd: +; CHECK: vmovsd (%rcx), %[[XMM:xmm[0-9]+]] +; CHECK-NEXT: vfmsub213sd (%rdx), %[[XMM]], %[[XMM]] +; CHECK-NEXT: vmovlpd %[[XMM]], (%rcx) +; CHECK-NEXT: ret + %a.val = load double, double* %a + %av0 = insertelement <2 x double> undef, double %a.val, i32 0 + %av = insertelement <2 x double> %av0, double 0.000000e+00, i32 1 + + %b.val = load double, double* %b + %bv0 = insertelement <2 x double> undef, double %b.val, i32 0 + %bv = insertelement <2 x double> %bv0, double 0.000000e+00, i32 1 + + %vr = call <2 x double> @llvm.x86.fma.vfmsub.sd(<2 x double> %av, <2 x double> %av, <2 x double> %bv) + + %sr = extractelement <2 x double> %vr, i32 0 + store double %sr, double* %a + ret void +} + +define void @fmsub_aba_sd(double* %a, double* %b) #0 { +; CHECK-LABEL: fmsub_aba_sd: +; CHECK: vmovsd (%rcx), %[[XMM:xmm[0-9]+]] +; CHECK-NEXT: vfmsub132sd (%rdx), %[[XMM]], %[[XMM]] +; CHECK-NEXT: vmovlpd %[[XMM]], (%rcx) +; CHECK-NEXT: ret + %a.val = load double, double* %a + %av0 = insertelement <2 x double> undef, double %a.val, i32 0 + %av = insertelement <2 x double> %av0, double 0.000000e+00, i32 1 + + %b.val = load double, double* %b + %bv0 = insertelement <2 x double> undef, double %b.val, i32 0 + %bv = insertelement <2 x double> %bv0, double 0.000000e+00, i32 1 + + %vr = call <2 x double> @llvm.x86.fma.vfmsub.sd(<2 x double> %av, <2 x double> %bv, <2 x double> %av) + + %sr = extractelement <2 x double> %vr, i32 0 + store double %sr, double* %a + ret void +} + +define void @fnmadd_aab_sd(double* %a, double* %b) #0 { +; CHECK-LABEL: fnmadd_aab_sd: +; CHECK: vmovsd (%rcx), %[[XMM:xmm[0-9]+]] +; CHECK-NEXT: vfnmadd213sd (%rdx), %[[XMM]], %[[XMM]] +; CHECK-NEXT: vmovlpd %[[XMM]], (%rcx) +; CHECK-NEXT: ret + %a.val = load double, double* %a + %av0 = insertelement <2 x double> undef, double %a.val, i32 0 + %av = insertelement <2 x double> %av0, double 0.000000e+00, i32 1 + + %b.val = load double, double* %b + %bv0 = insertelement <2 x double> undef, double %b.val, i32 0 + %bv = insertelement <2 x double> %bv0, double 0.000000e+00, i32 1 + + %vr = call <2 x double> @llvm.x86.fma.vfnmadd.sd(<2 x double> %av, <2 x double> %av, <2 x double> %bv) + + %sr = extractelement <2 x double> %vr, i32 0 + store double %sr, double* %a + ret void +} + +define void @fnmadd_aba_sd(double* %a, double* %b) #0 { +; CHECK-LABEL: fnmadd_aba_sd: +; CHECK: vmovsd (%rcx), %[[XMM:xmm[0-9]+]] +; CHECK-NEXT: vfnmadd132sd (%rdx), %[[XMM]], %[[XMM]] +; CHECK-NEXT: vmovlpd %[[XMM]], (%rcx) +; CHECK-NEXT: ret + %a.val = load double, double* %a + %av0 = insertelement <2 x double> undef, double %a.val, i32 0 + %av = insertelement <2 x double> %av0, double 0.000000e+00, i32 1 + + %b.val = load double, double* %b + %bv0 = insertelement <2 x double> undef, double %b.val, i32 0 + %bv = insertelement <2 x double> %bv0, double 0.000000e+00, i32 1 + + %vr = call <2 x double> @llvm.x86.fma.vfnmadd.sd(<2 x double> %av, <2 x double> %bv, <2 x double> %av) + + %sr = extractelement <2 x double> %vr, i32 0 + store double %sr, double* %a + ret void +} + +define void @fnmsub_aab_sd(double* %a, double* %b) #0 { +; CHECK-LABEL: fnmsub_aab_sd: +; CHECK: vmovsd (%rcx), %[[XMM:xmm[0-9]+]] +; CHECK-NEXT: vfnmsub213sd (%rdx), %[[XMM]], %[[XMM]] +; CHECK-NEXT: vmovlpd %[[XMM]], (%rcx) +; CHECK-NEXT: ret + %a.val = load double, double* %a + %av0 = insertelement <2 x double> undef, double %a.val, i32 0 + %av = insertelement <2 x double> %av0, double 0.000000e+00, i32 1 + + %b.val = load double, double* %b + %bv0 = insertelement <2 x double> undef, double %b.val, i32 0 + %bv = insertelement <2 x double> %bv0, double 0.000000e+00, i32 1 + + %vr = call <2 x double> @llvm.x86.fma.vfnmsub.sd(<2 x double> %av, <2 x double> %av, <2 x double> %bv) + + %sr = extractelement <2 x double> %vr, i32 0 + store double %sr, double* %a + ret void +} + +define void @fnmsub_aba_sd(double* %a, double* %b) #0 { +; CHECK-LABEL: fnmsub_aba_sd: +; CHECK: vmovsd (%rcx), %[[XMM:xmm[0-9]+]] +; CHECK-NEXT: vfnmsub132sd (%rdx), %[[XMM]], %[[XMM]] +; CHECK-NEXT: vmovlpd %[[XMM]], (%rcx) +; CHECK-NEXT: ret + %a.val = load double, double* %a + %av0 = insertelement <2 x double> undef, double %a.val, i32 0 + %av = insertelement <2 x double> %av0, double 0.000000e+00, i32 1 + + %b.val = load double, double* %b + %bv0 = insertelement <2 x double> undef, double %b.val, i32 0 + %bv = insertelement <2 x double> %bv0, double 0.000000e+00, i32 1 + + %vr = call <2 x double> @llvm.x86.fma.vfnmsub.sd(<2 x double> %av, <2 x double> %bv, <2 x double> %av) + + %sr = extractelement <2 x double> %vr, i32 0 + store double %sr, double* %a + ret void +} + + diff --git a/test/CodeGen/X86/fma_patterns.ll b/test/CodeGen/X86/fma_patterns.ll index a27b760face7..76a4acf00f90 100644 --- a/test/CodeGen/X86/fma_patterns.ll +++ b/test/CodeGen/X86/fma_patterns.ll @@ -1,212 +1,1195 @@ -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=core-avx2 -mattr=avx2,+fma -fp-contract=fast | FileCheck %s -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=bdver2 -mattr=-fma4 -fp-contract=fast | FileCheck %s -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=bdver1 -fp-contract=fast | FileCheck %s --check-prefix=CHECK_FMA4 - -; CHECK: test_x86_fmadd_ps -; CHECK: vfmadd213ps %xmm2, %xmm1, %xmm0 -; CHECK: ret -; CHECK_FMA4: test_x86_fmadd_ps -; CHECK_FMA4: vfmaddps %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK_FMA4: ret -define <4 x float> @test_x86_fmadd_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) { - %x = fmul <4 x float> %a0, %a1 - %res = fadd <4 x float> %x, %a2 - ret <4 x float> %res -} +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+fma -fp-contract=fast | FileCheck %s --check-prefix=ALL --check-prefix=FMA +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+fma4,+fma -fp-contract=fast | FileCheck %s --check-prefix=ALL --check-prefix=FMA4 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+fma4 -fp-contract=fast | FileCheck %s --check-prefix=ALL --check-prefix=FMA4 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq,+avx512vl -fp-contract=fast | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 -; CHECK: test_x86_fmsub_ps -; CHECK: fmsub213ps %xmm2, %xmm1, %xmm0 -; CHECK: ret -; CHECK_FMA4: test_x86_fmsub_ps -; CHECK_FMA4: vfmsubps %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK_FMA4: ret -define <4 x float> @test_x86_fmsub_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) { - %x = fmul <4 x float> %a0, %a1 - %res = fsub <4 x float> %x, %a2 - ret <4 x float> %res -} +; +; Pattern: (fadd (fmul x, y), z) -> (fmadd x,y,z) +; -; CHECK: test_x86_fnmadd_ps -; CHECK: fnmadd213ps %xmm2, %xmm1, %xmm0 -; CHECK: ret -; CHECK_FMA4: test_x86_fnmadd_ps -; CHECK_FMA4: vfnmaddps %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK_FMA4: ret -define <4 x float> @test_x86_fnmadd_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) { - %x = fmul <4 x float> %a0, %a1 - %res = fsub <4 x float> %a2, %x - ret <4 x float> %res +define float @test_f32_fmadd(float %a0, float %a1, float %a2) { +; FMA-LABEL: test_f32_fmadd: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213ss %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_f32_fmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddss %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_f32_fmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213ss %xmm2, %xmm0, %xmm1 +; AVX512-NEXT: vmovaps %zmm1, %zmm0 +; AVX512-NEXT: retq + %x = fmul float %a0, %a1 + %res = fadd float %x, %a2 + ret float %res } -; CHECK: test_x86_fnmsub_ps -; CHECK: fnmsub213ps %xmm2, %xmm1, %xmm0 -; CHECK: ret -; CHECK_FMA4: test_x86_fnmsub_ps -; CHECK_FMA4: fnmsubps %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK_FMA4: ret -define <4 x float> @test_x86_fnmsub_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) { +define <4 x float> @test_4f32_fmadd(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) { +; FMA-LABEL: test_4f32_fmadd: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213ps %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_4f32_fmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddps %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_4f32_fmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213ps %xmm2, %xmm1, %xmm0 +; AVX512-NEXT: retq %x = fmul <4 x float> %a0, %a1 - %y = fsub <4 x float> , %x - %res = fsub <4 x float> %y, %a2 + %res = fadd <4 x float> %x, %a2 ret <4 x float> %res } -; CHECK: test_x86_fmadd_ps_y -; CHECK: vfmadd213ps %ymm2, %ymm1, %ymm0 -; CHECK: ret -; CHECK_FMA4: test_x86_fmadd_ps_y -; CHECK_FMA4: vfmaddps %ymm2, %ymm1, %ymm0, %ymm0 -; CHECK_FMA4: ret -define <8 x float> @test_x86_fmadd_ps_y(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) { +define <8 x float> @test_8f32_fmadd(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) { +; FMA-LABEL: test_8f32_fmadd: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213ps %ymm2, %ymm1, %ymm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_8f32_fmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddps %ymm2, %ymm1, %ymm0, %ymm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_8f32_fmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213ps %ymm2, %ymm1, %ymm0 +; AVX512-NEXT: retq %x = fmul <8 x float> %a0, %a1 %res = fadd <8 x float> %x, %a2 ret <8 x float> %res } -; CHECK: test_x86_fmsub_ps_y -; CHECK: vfmsub213ps %ymm2, %ymm1, %ymm0 -; CHECK: ret -; CHECK_FMA4: test_x86_fmsub_ps_y -; CHECK_FMA4: vfmsubps %ymm2, %ymm1, %ymm0, %ymm0 -; CHECK_FMA4: ret -define <8 x float> @test_x86_fmsub_ps_y(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) { - %x = fmul <8 x float> %a0, %a1 - %res = fsub <8 x float> %x, %a2 - ret <8 x float> %res -} - -; CHECK: test_x86_fnmadd_ps_y -; CHECK: vfnmadd213ps %ymm2, %ymm1, %ymm0 -; CHECK: ret -; CHECK_FMA4: test_x86_fnmadd_ps_y -; CHECK_FMA4: vfnmaddps %ymm2, %ymm1, %ymm0, %ymm0 -; CHECK_FMA4: ret -define <8 x float> @test_x86_fnmadd_ps_y(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) { - %x = fmul <8 x float> %a0, %a1 - %res = fsub <8 x float> %a2, %x - ret <8 x float> %res +define double @test_f64_fmadd(double %a0, double %a1, double %a2) { +; FMA-LABEL: test_f64_fmadd: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213sd %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_f64_fmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddsd %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_f64_fmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213sd %xmm2, %xmm0, %xmm1 +; AVX512-NEXT: vmovaps %zmm1, %zmm0 +; AVX512-NEXT: retq + %x = fmul double %a0, %a1 + %res = fadd double %x, %a2 + ret double %res } -; CHECK: test_x86_fnmsub_ps_y -; CHECK: vfnmsub213ps %ymm2, %ymm1, %ymm0 -; CHECK: ret -define <8 x float> @test_x86_fnmsub_ps_y(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) { - %x = fmul <8 x float> %a0, %a1 - %y = fsub <8 x float> , %x - %res = fsub <8 x float> %y, %a2 - ret <8 x float> %res +define <2 x double> @test_2f64_fmadd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) { +; FMA-LABEL: test_2f64_fmadd: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213pd %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_2f64_fmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddpd %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_2f64_fmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213pd %xmm2, %xmm1, %xmm0 +; AVX512-NEXT: retq + %x = fmul <2 x double> %a0, %a1 + %res = fadd <2 x double> %x, %a2 + ret <2 x double> %res } -; CHECK: test_x86_fmadd_pd_y -; CHECK: vfmadd213pd %ymm2, %ymm1, %ymm0 -; CHECK: ret -; CHECK_FMA4: test_x86_fmadd_pd_y -; CHECK_FMA4: vfmaddpd %ymm2, %ymm1, %ymm0, %ymm0 -; CHECK_FMA4: ret -define <4 x double> @test_x86_fmadd_pd_y(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) { +define <4 x double> @test_4f64_fmadd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) { +; FMA-LABEL: test_4f64_fmadd: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213pd %ymm2, %ymm1, %ymm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_4f64_fmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddpd %ymm2, %ymm1, %ymm0, %ymm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_4f64_fmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213pd %ymm2, %ymm1, %ymm0 +; AVX512-NEXT: retq %x = fmul <4 x double> %a0, %a1 %res = fadd <4 x double> %x, %a2 ret <4 x double> %res } -; CHECK: test_x86_fmsub_pd_y -; CHECK: vfmsub213pd %ymm2, %ymm1, %ymm0 -; CHECK: ret -; CHECK_FMA4: test_x86_fmsub_pd_y -; CHECK_FMA4: vfmsubpd %ymm2, %ymm1, %ymm0, %ymm0 -; CHECK_FMA4: ret -define <4 x double> @test_x86_fmsub_pd_y(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) { - %x = fmul <4 x double> %a0, %a1 - %res = fsub <4 x double> %x, %a2 - ret <4 x double> %res +; +; Pattern: (fsub (fmul x, y), z) -> (fmsub x, y, z) +; + +define float @test_f32_fmsub(float %a0, float %a1, float %a2) { +; FMA-LABEL: test_f32_fmsub: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213ss %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_f32_fmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubss %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_f32_fmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213ss %xmm2, %xmm0, %xmm1 +; AVX512-NEXT: vmovaps %zmm1, %zmm0 +; AVX512-NEXT: retq + %x = fmul float %a0, %a1 + %res = fsub float %x, %a2 + ret float %res +} + +define <4 x float> @test_4f32_fmsub(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) { +; FMA-LABEL: test_4f32_fmsub: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213ps %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_4f32_fmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubps %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_4f32_fmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213ps %xmm2, %xmm1, %xmm0 +; AVX512-NEXT: retq + %x = fmul <4 x float> %a0, %a1 + %res = fsub <4 x float> %x, %a2 + ret <4 x float> %res } -; CHECK: test_x86_fmsub_pd -; CHECK: vfmsub213pd %xmm2, %xmm1, %xmm0 -; CHECK: ret -; CHECK_FMA4: test_x86_fmsub_pd -; CHECK_FMA4: vfmsubpd %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK_FMA4: ret -define <2 x double> @test_x86_fmsub_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) { +define <8 x float> @test_8f32_fmsub(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) { +; FMA-LABEL: test_8f32_fmsub: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213ps %ymm2, %ymm1, %ymm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_8f32_fmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubps %ymm2, %ymm1, %ymm0, %ymm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_8f32_fmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213ps %ymm2, %ymm1, %ymm0 +; AVX512-NEXT: retq + %x = fmul <8 x float> %a0, %a1 + %res = fsub <8 x float> %x, %a2 + ret <8 x float> %res +} + +define double @test_f64_fmsub(double %a0, double %a1, double %a2) { +; FMA-LABEL: test_f64_fmsub: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213sd %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_f64_fmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubsd %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_f64_fmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213sd %xmm2, %xmm0, %xmm1 +; AVX512-NEXT: vmovaps %zmm1, %zmm0 +; AVX512-NEXT: retq + %x = fmul double %a0, %a1 + %res = fsub double %x, %a2 + ret double %res +} + +define <2 x double> @test_2f64_fmsub(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) { +; FMA-LABEL: test_2f64_fmsub: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213pd %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_2f64_fmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubpd %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_2f64_fmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213pd %xmm2, %xmm1, %xmm0 +; AVX512-NEXT: retq %x = fmul <2 x double> %a0, %a1 %res = fsub <2 x double> %x, %a2 ret <2 x double> %res } -; CHECK: test_x86_fnmadd_ss -; CHECK: vfnmadd213ss %xmm2, %xmm1, %xmm0 -; CHECK: ret -; CHECK_FMA4: test_x86_fnmadd_ss -; CHECK_FMA4: vfnmaddss %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK_FMA4: ret -define float @test_x86_fnmadd_ss(float %a0, float %a1, float %a2) { +define <4 x double> @test_4f64_fmsub(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) { +; FMA-LABEL: test_4f64_fmsub: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213pd %ymm2, %ymm1, %ymm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_4f64_fmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubpd %ymm2, %ymm1, %ymm0, %ymm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_4f64_fmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213pd %ymm2, %ymm1, %ymm0 +; AVX512-NEXT: retq + %x = fmul <4 x double> %a0, %a1 + %res = fsub <4 x double> %x, %a2 + ret <4 x double> %res +} + +; +; Pattern: (fsub z, (fmul x, y)) -> (fnmadd x, y, z) +; + +define float @test_f32_fnmadd(float %a0, float %a1, float %a2) { +; FMA-LABEL: test_f32_fnmadd: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213ss %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_f32_fnmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddss %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_f32_fnmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmadd213ss %xmm2, %xmm0, %xmm1 +; AVX512-NEXT: vmovaps %zmm1, %zmm0 +; AVX512-NEXT: retq %x = fmul float %a0, %a1 %res = fsub float %a2, %x ret float %res } -; CHECK: test_x86_fnmadd_sd -; CHECK: vfnmadd213sd %xmm2, %xmm1, %xmm0 -; CHECK: ret -; CHECK_FMA4: test_x86_fnmadd_sd -; CHECK_FMA4: vfnmaddsd %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK_FMA4: ret -define double @test_x86_fnmadd_sd(double %a0, double %a1, double %a2) { +define <4 x float> @test_4f32_fnmadd(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) { +; FMA-LABEL: test_4f32_fnmadd: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213ps %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_4f32_fnmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddps %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_4f32_fnmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmadd213ps %xmm2, %xmm1, %xmm0 +; AVX512-NEXT: retq + %x = fmul <4 x float> %a0, %a1 + %res = fsub <4 x float> %a2, %x + ret <4 x float> %res +} + +define <8 x float> @test_8f32_fnmadd(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) { +; FMA-LABEL: test_8f32_fnmadd: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213ps %ymm2, %ymm1, %ymm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_8f32_fnmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddps %ymm2, %ymm1, %ymm0, %ymm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_8f32_fnmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmadd213ps %ymm2, %ymm1, %ymm0 +; AVX512-NEXT: retq + %x = fmul <8 x float> %a0, %a1 + %res = fsub <8 x float> %a2, %x + ret <8 x float> %res +} + +define double @test_f64_fnmadd(double %a0, double %a1, double %a2) { +; FMA-LABEL: test_f64_fnmadd: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213sd %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_f64_fnmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddsd %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_f64_fnmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmadd213sd %xmm2, %xmm0, %xmm1 +; AVX512-NEXT: vmovaps %zmm1, %zmm0 +; AVX512-NEXT: retq %x = fmul double %a0, %a1 %res = fsub double %a2, %x ret double %res } -; CHECK: test_x86_fmsub_sd -; CHECK: vfmsub213sd %xmm2, %xmm1, %xmm0 -; CHECK: ret -; CHECK_FMA4: test_x86_fmsub_sd -; CHECK_FMA4: vfmsubsd %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK_FMA4: ret -define double @test_x86_fmsub_sd(double %a0, double %a1, double %a2) { - %x = fmul double %a0, %a1 - %res = fsub double %x, %a2 - ret double %res +define <2 x double> @test_2f64_fnmadd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) { +; FMA-LABEL: test_2f64_fnmadd: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213pd %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_2f64_fnmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddpd %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_2f64_fnmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmadd213pd %xmm2, %xmm1, %xmm0 +; AVX512-NEXT: retq + %x = fmul <2 x double> %a0, %a1 + %res = fsub <2 x double> %a2, %x + ret <2 x double> %res } -; CHECK: test_x86_fnmsub_ss -; CHECK: vfnmsub213ss %xmm2, %xmm1, %xmm0 -; CHECK: ret -; CHECK_FMA4: test_x86_fnmsub_ss -; CHECK_FMA4: vfnmsubss %xmm2, %xmm1, %xmm0, %xmm0 -; CHECK_FMA4: ret -define float @test_x86_fnmsub_ss(float %a0, float %a1, float %a2) { - %x = fsub float -0.000000e+00, %a0 - %y = fmul float %x, %a1 +define <4 x double> @test_4f64_fnmadd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) { +; FMA-LABEL: test_4f64_fnmadd: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213pd %ymm2, %ymm1, %ymm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_4f64_fnmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddpd %ymm2, %ymm1, %ymm0, %ymm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_4f64_fnmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmadd213pd %ymm2, %ymm1, %ymm0 +; AVX512-NEXT: retq + %x = fmul <4 x double> %a0, %a1 + %res = fsub <4 x double> %a2, %x + ret <4 x double> %res +} + +; +; Pattern: (fsub (fneg (fmul x, y)), z) -> (fnmsub x, y, z) +; + +define float @test_f32_fnmsub(float %a0, float %a1, float %a2) { +; FMA-LABEL: test_f32_fnmsub: +; FMA: # BB#0: +; FMA-NEXT: vfnmsub213ss %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_f32_fnmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmsubss %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_f32_fnmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmsub213ss %xmm2, %xmm0, %xmm1 +; AVX512-NEXT: vmovaps %zmm1, %zmm0 +; AVX512-NEXT: retq + %x = fmul float %a0, %a1 + %y = fsub float -0.000000e+00, %x %res = fsub float %y, %a2 ret float %res } -; CHECK: test_x86_fmadd_ps_load -; CHECK: vmovaps (%rdi), %xmm2 -; CHECK: vfmadd213ps %xmm1, %xmm2, %xmm0 -; CHECK: ret -; CHECK_FMA4: test_x86_fmadd_ps_load -; CHECK_FMA4: vfmaddps %xmm1, (%rdi), %xmm0, %xmm0 -; CHECK_FMA4: ret -define <4 x float> @test_x86_fmadd_ps_load(<4 x float>* %a0, <4 x float> %a1, <4 x float> %a2) { - %x = load <4 x float>, <4 x float>* %a0 - %y = fmul <4 x float> %x, %a1 - %res = fadd <4 x float> %y, %a2 +define <4 x float> @test_4f32_fnmsub(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) { +; FMA-LABEL: test_4f32_fnmsub: +; FMA: # BB#0: +; FMA-NEXT: vfnmsub213ps %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_4f32_fnmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmsubps %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_4f32_fnmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmsub213ps %xmm2, %xmm1, %xmm0 +; AVX512-NEXT: retq + %x = fmul <4 x float> %a0, %a1 + %y = fsub <4 x float> , %x + %res = fsub <4 x float> %y, %a2 ret <4 x float> %res } -; CHECK: test_x86_fmsub_ps_load -; CHECK: vmovaps (%rdi), %xmm2 -; CHECK: fmsub213ps %xmm1, %xmm2, %xmm0 -; CHECK: ret -; CHECK_FMA4: test_x86_fmsub_ps_load -; CHECK_FMA4: vfmsubps %xmm1, (%rdi), %xmm0, %xmm0 -; CHECK_FMA4: ret -define <4 x float> @test_x86_fmsub_ps_load(<4 x float>* %a0, <4 x float> %a1, <4 x float> %a2) { +define <8 x float> @test_8f32_fnmsub(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) { +; FMA-LABEL: test_8f32_fnmsub: +; FMA: # BB#0: +; FMA-NEXT: vfnmsub213ps %ymm2, %ymm1, %ymm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_8f32_fnmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmsubps %ymm2, %ymm1, %ymm0, %ymm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_8f32_fnmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmsub213ps %ymm2, %ymm1, %ymm0 +; AVX512-NEXT: retq + %x = fmul <8 x float> %a0, %a1 + %y = fsub <8 x float> , %x + %res = fsub <8 x float> %y, %a2 + ret <8 x float> %res +} + +define double @test_f64_fnmsub(double %a0, double %a1, double %a2) { +; FMA-LABEL: test_f64_fnmsub: +; FMA: # BB#0: +; FMA-NEXT: vfnmsub213sd %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_f64_fnmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmsubsd %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_f64_fnmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmsub213sd %xmm2, %xmm0, %xmm1 +; AVX512-NEXT: vmovaps %zmm1, %zmm0 +; AVX512-NEXT: retq + %x = fmul double %a0, %a1 + %y = fsub double -0.000000e+00, %x + %res = fsub double %y, %a2 + ret double %res +} + +define <2 x double> @test_2f64_fnmsub(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) { +; FMA-LABEL: test_2f64_fnmsub: +; FMA: # BB#0: +; FMA-NEXT: vfnmsub213pd %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_2f64_fnmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmsubpd %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_2f64_fnmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmsub213pd %xmm2, %xmm1, %xmm0 +; AVX512-NEXT: retq + %x = fmul <2 x double> %a0, %a1 + %y = fsub <2 x double> , %x + %res = fsub <2 x double> %y, %a2 + ret <2 x double> %res +} + +define <4 x double> @test_4f64_fnmsub(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) { +; FMA-LABEL: test_4f64_fnmsub: +; FMA: # BB#0: +; FMA-NEXT: vfnmsub213pd %ymm2, %ymm1, %ymm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_4f64_fnmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmsubpd %ymm2, %ymm1, %ymm0, %ymm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_4f64_fnmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmsub213pd %ymm2, %ymm1, %ymm0 +; AVX512-NEXT: retq + %x = fmul <4 x double> %a0, %a1 + %y = fsub <4 x double> , %x + %res = fsub <4 x double> %y, %a2 + ret <4 x double> %res +} + +; +; Load Folding Patterns +; + +define <4 x float> @test_4f32_fmadd_load(<4 x float>* %a0, <4 x float> %a1, <4 x float> %a2) { +; FMA-LABEL: test_4f32_fmadd_load: +; FMA: # BB#0: +; FMA-NEXT: vfmadd132ps (%rdi), %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_4f32_fmadd_load: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddps %xmm1, (%rdi), %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_4f32_fmadd_load: +; AVX512: # BB#0: +; AVX512-NEXT: vmovaps (%rdi), %xmm2 +; AVX512-NEXT: vfmadd213ps %xmm1, %xmm0, %xmm2 +; AVX512-NEXT: vmovaps %zmm2, %zmm0 +; AVX512-NEXT: retq %x = load <4 x float>, <4 x float>* %a0 %y = fmul <4 x float> %x, %a1 - %res = fsub <4 x float> %y, %a2 + %res = fadd <4 x float> %y, %a2 ret <4 x float> %res } +define <2 x double> @test_2f64_fmsub_load(<2 x double>* %a0, <2 x double> %a1, <2 x double> %a2) { +; FMA-LABEL: test_2f64_fmsub_load: +; FMA: # BB#0: +; FMA-NEXT: vfmsub132pd (%rdi), %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_2f64_fmsub_load: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubpd %xmm1, (%rdi), %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_2f64_fmsub_load: +; AVX512: # BB#0: +; AVX512-NEXT: vmovapd (%rdi), %xmm2 +; AVX512-NEXT: vfmsub213pd %xmm1, %xmm0, %xmm2 +; AVX512-NEXT: vmovaps %zmm2, %zmm0 +; AVX512-NEXT: retq + %x = load <2 x double>, <2 x double>* %a0 + %y = fmul <2 x double> %x, %a1 + %res = fsub <2 x double> %y, %a2 + ret <2 x double> %res +} + +; +; Patterns (+ fneg variants): mul(add(1.0,x),y), mul(sub(1.0,x),y), mul(sub(x,1.0),y) +; + +define <4 x float> @test_v4f32_mul_add_x_one_y(<4 x float> %x, <4 x float> %y) { +; FMA-LABEL: test_v4f32_mul_add_x_one_y: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213ps %xmm1, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_mul_add_x_one_y: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddps %xmm1, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_mul_add_x_one_y: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213ps %xmm1, %xmm1, %xmm0 +; AVX512-NEXT: retq + %a = fadd <4 x float> %x, + %m = fmul <4 x float> %a, %y + ret <4 x float> %m +} + +define <4 x float> @test_v4f32_mul_y_add_x_one(<4 x float> %x, <4 x float> %y) { +; FMA-LABEL: test_v4f32_mul_y_add_x_one: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213ps %xmm1, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_mul_y_add_x_one: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddps %xmm1, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_mul_y_add_x_one: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213ps %xmm1, %xmm1, %xmm0 +; AVX512-NEXT: retq + %a = fadd <4 x float> %x, + %m = fmul <4 x float> %y, %a + ret <4 x float> %m +} + +define <4 x float> @test_v4f32_mul_add_x_negone_y(<4 x float> %x, <4 x float> %y) { +; FMA-LABEL: test_v4f32_mul_add_x_negone_y: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213ps %xmm1, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_mul_add_x_negone_y: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubps %xmm1, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_mul_add_x_negone_y: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213ps %xmm1, %xmm1, %xmm0 +; AVX512-NEXT: retq + %a = fadd <4 x float> %x, + %m = fmul <4 x float> %a, %y + ret <4 x float> %m +} + +define <4 x float> @test_v4f32_mul_y_add_x_negone(<4 x float> %x, <4 x float> %y) { +; FMA-LABEL: test_v4f32_mul_y_add_x_negone: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213ps %xmm1, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_mul_y_add_x_negone: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubps %xmm1, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_mul_y_add_x_negone: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213ps %xmm1, %xmm1, %xmm0 +; AVX512-NEXT: retq + %a = fadd <4 x float> %x, + %m = fmul <4 x float> %y, %a + ret <4 x float> %m +} + +define <4 x float> @test_v4f32_mul_sub_one_x_y(<4 x float> %x, <4 x float> %y) { +; FMA-LABEL: test_v4f32_mul_sub_one_x_y: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213ps %xmm1, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_mul_sub_one_x_y: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddps %xmm1, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_mul_sub_one_x_y: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmadd213ps %xmm1, %xmm1, %xmm0 +; AVX512-NEXT: retq + %s = fsub <4 x float> , %x + %m = fmul <4 x float> %s, %y + ret <4 x float> %m +} + +define <4 x float> @test_v4f32_mul_y_sub_one_x(<4 x float> %x, <4 x float> %y) { +; FMA-LABEL: test_v4f32_mul_y_sub_one_x: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213ps %xmm1, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_mul_y_sub_one_x: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddps %xmm1, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_mul_y_sub_one_x: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmadd213ps %xmm1, %xmm1, %xmm0 +; AVX512-NEXT: retq + %s = fsub <4 x float> , %x + %m = fmul <4 x float> %y, %s + ret <4 x float> %m +} + +define <4 x float> @test_v4f32_mul_sub_negone_x_y(<4 x float> %x, <4 x float> %y) { +; FMA-LABEL: test_v4f32_mul_sub_negone_x_y: +; FMA: # BB#0: +; FMA-NEXT: vfnmsub213ps %xmm1, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_mul_sub_negone_x_y: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmsubps %xmm1, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_mul_sub_negone_x_y: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmsub213ps %xmm1, %xmm1, %xmm0 +; AVX512-NEXT: retq + %s = fsub <4 x float> , %x + %m = fmul <4 x float> %s, %y + ret <4 x float> %m +} + +define <4 x float> @test_v4f32_mul_y_sub_negone_x(<4 x float> %x, <4 x float> %y) { +; FMA-LABEL: test_v4f32_mul_y_sub_negone_x: +; FMA: # BB#0: +; FMA-NEXT: vfnmsub213ps %xmm1, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_mul_y_sub_negone_x: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmsubps %xmm1, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_mul_y_sub_negone_x: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmsub213ps %xmm1, %xmm1, %xmm0 +; AVX512-NEXT: retq + %s = fsub <4 x float> , %x + %m = fmul <4 x float> %y, %s + ret <4 x float> %m +} + +define <4 x float> @test_v4f32_mul_sub_x_one_y(<4 x float> %x, <4 x float> %y) { +; FMA-LABEL: test_v4f32_mul_sub_x_one_y: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213ps %xmm1, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_mul_sub_x_one_y: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubps %xmm1, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_mul_sub_x_one_y: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213ps %xmm1, %xmm1, %xmm0 +; AVX512-NEXT: retq + %s = fsub <4 x float> %x, + %m = fmul <4 x float> %s, %y + ret <4 x float> %m +} + +define <4 x float> @test_v4f32_mul_y_sub_x_one(<4 x float> %x, <4 x float> %y) { +; FMA-LABEL: test_v4f32_mul_y_sub_x_one: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213ps %xmm1, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_mul_y_sub_x_one: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubps %xmm1, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_mul_y_sub_x_one: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213ps %xmm1, %xmm1, %xmm0 +; AVX512-NEXT: retq + %s = fsub <4 x float> %x, + %m = fmul <4 x float> %y, %s + ret <4 x float> %m +} + +define <4 x float> @test_v4f32_mul_sub_x_negone_y(<4 x float> %x, <4 x float> %y) { +; FMA-LABEL: test_v4f32_mul_sub_x_negone_y: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213ps %xmm1, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_mul_sub_x_negone_y: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddps %xmm1, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_mul_sub_x_negone_y: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213ps %xmm1, %xmm1, %xmm0 +; AVX512-NEXT: retq + %s = fsub <4 x float> %x, + %m = fmul <4 x float> %s, %y + ret <4 x float> %m +} + +define <4 x float> @test_v4f32_mul_y_sub_x_negone(<4 x float> %x, <4 x float> %y) { +; FMA-LABEL: test_v4f32_mul_y_sub_x_negone: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213ps %xmm1, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_mul_y_sub_x_negone: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddps %xmm1, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_mul_y_sub_x_negone: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213ps %xmm1, %xmm1, %xmm0 +; AVX512-NEXT: retq + %s = fsub <4 x float> %x, + %m = fmul <4 x float> %y, %s + ret <4 x float> %m +} + +; +; Interpolation Patterns: add(mul(x,t),mul(sub(1.0,t),y)) +; + +define float @test_f32_interp(float %x, float %y, float %t) { +; FMA-LABEL: test_f32_interp: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213ss %xmm1, %xmm2, %xmm1 +; FMA-NEXT: vfmadd213ss %xmm1, %xmm2, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_f32_interp: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddss %xmm1, %xmm1, %xmm2, %xmm1 +; FMA4-NEXT: vfmaddss %xmm1, %xmm2, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_f32_interp: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmadd213ss %xmm1, %xmm2, %xmm1 +; AVX512-NEXT: vfmadd213ss %xmm1, %xmm0, %xmm2 +; AVX512-NEXT: vmovaps %zmm2, %zmm0 +; AVX512-NEXT: retq + %t1 = fsub float 1.0, %t + %tx = fmul float %x, %t + %ty = fmul float %y, %t1 + %r = fadd float %tx, %ty + ret float %r +} + +define <4 x float> @test_v4f32_interp(<4 x float> %x, <4 x float> %y, <4 x float> %t) { +; FMA-LABEL: test_v4f32_interp: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213ps %xmm1, %xmm2, %xmm1 +; FMA-NEXT: vfmadd213ps %xmm1, %xmm2, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_interp: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddps %xmm1, %xmm1, %xmm2, %xmm1 +; FMA4-NEXT: vfmaddps %xmm1, %xmm2, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_interp: +; AVX512: # BB#0: +; AVX512-NEXT: vmovaps %zmm2, %zmm3 +; AVX512-NEXT: vfnmadd213ps %xmm1, %xmm1, %xmm3 +; AVX512-NEXT: vfmadd213ps %xmm3, %xmm2, %xmm0 +; AVX512-NEXT: retq + %t1 = fsub <4 x float> , %t + %tx = fmul <4 x float> %x, %t + %ty = fmul <4 x float> %y, %t1 + %r = fadd <4 x float> %tx, %ty + ret <4 x float> %r +} + +define <8 x float> @test_v8f32_interp(<8 x float> %x, <8 x float> %y, <8 x float> %t) { +; FMA-LABEL: test_v8f32_interp: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213ps %ymm1, %ymm2, %ymm1 +; FMA-NEXT: vfmadd213ps %ymm1, %ymm2, %ymm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v8f32_interp: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddps %ymm1, %ymm1, %ymm2, %ymm1 +; FMA4-NEXT: vfmaddps %ymm1, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v8f32_interp: +; AVX512: # BB#0: +; AVX512-NEXT: vmovaps %zmm2, %zmm3 +; AVX512-NEXT: vfnmadd213ps %ymm1, %ymm1, %ymm3 +; AVX512-NEXT: vfmadd213ps %ymm3, %ymm2, %ymm0 +; AVX512-NEXT: retq + %t1 = fsub <8 x float> , %t + %tx = fmul <8 x float> %x, %t + %ty = fmul <8 x float> %y, %t1 + %r = fadd <8 x float> %tx, %ty + ret <8 x float> %r +} + +define double @test_f64_interp(double %x, double %y, double %t) { +; FMA-LABEL: test_f64_interp: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213sd %xmm1, %xmm2, %xmm1 +; FMA-NEXT: vfmadd213sd %xmm1, %xmm2, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_f64_interp: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddsd %xmm1, %xmm1, %xmm2, %xmm1 +; FMA4-NEXT: vfmaddsd %xmm1, %xmm2, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_f64_interp: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmadd213sd %xmm1, %xmm2, %xmm1 +; AVX512-NEXT: vfmadd213sd %xmm1, %xmm0, %xmm2 +; AVX512-NEXT: vmovaps %zmm2, %zmm0 +; AVX512-NEXT: retq + %t1 = fsub double 1.0, %t + %tx = fmul double %x, %t + %ty = fmul double %y, %t1 + %r = fadd double %tx, %ty + ret double %r +} + +define <2 x double> @test_v2f64_interp(<2 x double> %x, <2 x double> %y, <2 x double> %t) { +; FMA-LABEL: test_v2f64_interp: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213pd %xmm1, %xmm2, %xmm1 +; FMA-NEXT: vfmadd213pd %xmm1, %xmm2, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v2f64_interp: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddpd %xmm1, %xmm1, %xmm2, %xmm1 +; FMA4-NEXT: vfmaddpd %xmm1, %xmm2, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v2f64_interp: +; AVX512: # BB#0: +; AVX512-NEXT: vmovaps %zmm2, %zmm3 +; AVX512-NEXT: vfnmadd213pd %xmm1, %xmm1, %xmm3 +; AVX512-NEXT: vfmadd213pd %xmm3, %xmm2, %xmm0 +; AVX512-NEXT: retq + %t1 = fsub <2 x double> , %t + %tx = fmul <2 x double> %x, %t + %ty = fmul <2 x double> %y, %t1 + %r = fadd <2 x double> %tx, %ty + ret <2 x double> %r +} + +define <4 x double> @test_v4f64_interp(<4 x double> %x, <4 x double> %y, <4 x double> %t) { +; FMA-LABEL: test_v4f64_interp: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213pd %ymm1, %ymm2, %ymm1 +; FMA-NEXT: vfmadd213pd %ymm1, %ymm2, %ymm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f64_interp: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddpd %ymm1, %ymm1, %ymm2, %ymm1 +; FMA4-NEXT: vfmaddpd %ymm1, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f64_interp: +; AVX512: # BB#0: +; AVX512-NEXT: vmovaps %zmm2, %zmm3 +; AVX512-NEXT: vfnmadd213pd %ymm1, %ymm1, %ymm3 +; AVX512-NEXT: vfmadd213pd %ymm3, %ymm2, %ymm0 +; AVX512-NEXT: retq + %t1 = fsub <4 x double> , %t + %tx = fmul <4 x double> %x, %t + %ty = fmul <4 x double> %y, %t1 + %r = fadd <4 x double> %tx, %ty + ret <4 x double> %r +} + +; +; Pattern: (fneg (fma x, y, z)) -> (fma x, -y, -z) +; + +define <4 x float> @test_v4f32_fneg_fmadd(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 { +; FMA-LABEL: test_v4f32_fneg_fmadd: +; FMA: # BB#0: +; FMA-NEXT: vfnmsub213ps %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_fneg_fmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmsubps %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_fneg_fmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmsub213ps %xmm2, %xmm1, %xmm0 +; AVX512-NEXT: retq + %mul = fmul <4 x float> %a0, %a1 + %add = fadd <4 x float> %mul, %a2 + %neg = fsub <4 x float> , %add + ret <4 x float> %neg +} + +define <4 x double> @test_v4f64_fneg_fmsub(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 { +; FMA-LABEL: test_v4f64_fneg_fmsub: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213pd %ymm2, %ymm1, %ymm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f64_fneg_fmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddpd %ymm2, %ymm1, %ymm0, %ymm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f64_fneg_fmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmadd213pd %ymm2, %ymm1, %ymm0 +; AVX512-NEXT: retq + %mul = fmul <4 x double> %a0, %a1 + %sub = fsub <4 x double> %mul, %a2 + %neg = fsub <4 x double> , %sub + ret <4 x double> %neg +} + +define <4 x float> @test_v4f32_fneg_fnmadd(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 { +; FMA-LABEL: test_v4f32_fneg_fnmadd: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213ps %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_fneg_fnmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubps %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_fneg_fnmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213ps %xmm2, %xmm1, %xmm0 +; AVX512-NEXT: retq + %mul = fmul <4 x float> %a0, %a1 + %neg0 = fsub <4 x float> , %mul + %add = fadd <4 x float> %neg0, %a2 + %neg1 = fsub <4 x float> , %add + ret <4 x float> %neg1 +} + +define <4 x double> @test_v4f64_fneg_fnmsub(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 { +; FMA-LABEL: test_v4f64_fneg_fnmsub: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213pd %ymm2, %ymm1, %ymm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f64_fneg_fnmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddpd %ymm2, %ymm1, %ymm0, %ymm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f64_fneg_fnmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213pd %ymm2, %ymm1, %ymm0 +; AVX512-NEXT: retq + %mul = fmul <4 x double> %a0, %a1 + %neg0 = fsub <4 x double> , %mul + %sub = fsub <4 x double> %neg0, %a2 + %neg1 = fsub <4 x double> , %sub + ret <4 x double> %neg1 +} + +; +; Pattern: (fma x, c1, (fmul x, c2)) -> (fmul x, c1+c2) +; + +define <4 x float> @test_v4f32_fma_x_c1_fmul_x_c2(<4 x float> %x) #0 { +; FMA-LABEL: test_v4f32_fma_x_c1_fmul_x_c2: +; FMA: # BB#0: +; FMA-NEXT: vmulps {{.*}}(%rip), %xmm0, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_fma_x_c1_fmul_x_c2: +; FMA4: # BB#0: +; FMA4-NEXT: vmulps {{.*}}(%rip), %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_fma_x_c1_fmul_x_c2: +; AVX512: # BB#0: +; AVX512-NEXT: vmulps {{.*}}(%rip){1to4}, %xmm0, %xmm0 +; AVX512-NEXT: retq + %m0 = fmul <4 x float> %x, + %m1 = fmul <4 x float> %x, + %a = fadd <4 x float> %m0, %m1 + ret <4 x float> %a +} + +; +; Pattern: (fma (fmul x, c1), c2, y) -> (fma x, c1*c2, y) +; + +define <4 x float> @test_v4f32_fma_fmul_x_c1_c2_y(<4 x float> %x, <4 x float> %y) #0 { +; FMA-LABEL: test_v4f32_fma_fmul_x_c1_c2_y: +; FMA: # BB#0: +; FMA-NEXT: vfmadd132ps {{.*}}(%rip), %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_fma_fmul_x_c1_c2_y: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddps %xmm1, {{.*}}(%rip), %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_fma_fmul_x_c1_c2_y: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd231ps {{.*}}(%rip), %xmm0, %xmm1 +; AVX512-NEXT: vmovaps %zmm1, %zmm0 +; AVX512-NEXT: retq + %m0 = fmul <4 x float> %x, + %m1 = fmul <4 x float> %m0, + %a = fadd <4 x float> %m1, %y + ret <4 x float> %a +} + +; Pattern: (fneg (fmul x, y)) -> (fnmsub x, y, 0) + +define double @test_f64_fneg_fmul(double %x, double %y) #0 { +; FMA-LABEL: test_f64_fneg_fmul: +; FMA: # BB#0: +; FMA-NEXT: vxorpd %xmm2, %xmm2, %xmm2 +; FMA-NEXT: vfnmsub213sd %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_f64_fneg_fmul: +; FMA4: # BB#0: +; FMA4-NEXT: vxorpd %xmm2, %xmm2, %xmm2 +; FMA4-NEXT: vfnmsubsd %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_f64_fneg_fmul: +; AVX512: # BB#0: +; AVX512-NEXT: vxorps %xmm2, %xmm2, %xmm2 +; AVX512-NEXT: vfnmsub213sd %xmm2, %xmm0, %xmm1 +; AVX512-NEXT: vmovaps %zmm1, %zmm0 +; AVX512-NEXT: retq + %m = fmul nsz double %x, %y + %n = fsub double -0.0, %m + ret double %n +} + +define <4 x float> @test_v4f32_fneg_fmul(<4 x float> %x, <4 x float> %y) #0 { +; FMA-LABEL: test_v4f32_fneg_fmul: +; FMA: # BB#0: +; FMA-NEXT: vxorps %xmm2, %xmm2, %xmm2 +; FMA-NEXT: vfnmsub213ps %xmm2, %xmm1, %xmm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f32_fneg_fmul: +; FMA4: # BB#0: +; FMA4-NEXT: vxorps %xmm2, %xmm2, %xmm2 +; FMA4-NEXT: vfnmsubps %xmm2, %xmm1, %xmm0, %xmm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f32_fneg_fmul: +; AVX512: # BB#0: +; AVX512-NEXT: vxorps %xmm2, %xmm2, %xmm2 +; AVX512-NEXT: vfnmsub213ps %xmm2, %xmm1, %xmm0 +; AVX512-NEXT: retq + %m = fmul nsz <4 x float> %x, %y + %n = fsub <4 x float> , %m + ret <4 x float> %n +} + +define <4 x double> @test_v4f64_fneg_fmul(<4 x double> %x, <4 x double> %y) #0 { +; FMA-LABEL: test_v4f64_fneg_fmul: +; FMA: # BB#0: +; FMA-NEXT: vxorpd %ymm2, %ymm2, %ymm2 +; FMA-NEXT: vfnmsub213pd %ymm2, %ymm1, %ymm0 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v4f64_fneg_fmul: +; FMA4: # BB#0: +; FMA4-NEXT: vxorpd %ymm2, %ymm2, %ymm2 +; FMA4-NEXT: vfnmsubpd %ymm2, %ymm1, %ymm0, %ymm0 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v4f64_fneg_fmul: +; AVX512: # BB#0: +; AVX512-NEXT: vxorps %ymm2, %ymm2, %ymm2 +; AVX512-NEXT: vfnmsub213pd %ymm2, %ymm1, %ymm0 +; AVX512-NEXT: retq + %m = fmul nsz <4 x double> %x, %y + %n = fsub <4 x double> , %m + ret <4 x double> %n +} + +define <4 x double> @test_v4f64_fneg_fmul_no_nsz(<4 x double> %x, <4 x double> %y) #0 { +; ALL-LABEL: test_v4f64_fneg_fmul_no_nsz: +; ALL: # BB#0: +; ALL-NEXT: vmulpd %ymm1, %ymm0, %ymm0 +; ALL-NEXT: vxorpd {{.*}}(%rip), %ymm0, %ymm0 +; ALL-NEXT: retq + %m = fmul <4 x double> %x, %y + %n = fsub <4 x double> , %m + ret <4 x double> %n +} + +attributes #0 = { "unsafe-fp-math"="true" } diff --git a/test/CodeGen/X86/fma_patterns_wide.ll b/test/CodeGen/X86/fma_patterns_wide.ll index 04db2d76cd8c..7b6509ad51c7 100644 --- a/test/CodeGen/X86/fma_patterns_wide.ll +++ b/test/CodeGen/X86/fma_patterns_wide.ll @@ -1,84 +1,821 @@ -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=core-avx2 -mattr=avx2,+fma -fp-contract=fast | FileCheck %s -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=bdver2 -mattr=-fma4 -fp-contract=fast | FileCheck %s -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=bdver1 -fp-contract=fast | FileCheck %s --check-prefix=CHECK_FMA4 - -; CHECK-LABEL: test_x86_fmadd_ps_y_wide -; CHECK: vfmadd213ps -; CHECK: vfmadd213ps -; CHECK: ret -; CHECK_FMA4-LABEL: test_x86_fmadd_ps_y_wide -; CHECK_FMA4: vfmaddps -; CHECK_FMA4: vfmaddps -; CHECK_FMA4: ret -define <16 x float> @test_x86_fmadd_ps_y_wide(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) { +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+fma -fp-contract=fast | FileCheck %s --check-prefix=FMA +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+fma4,+fma -fp-contract=fast | FileCheck %s --check-prefix=FMA4 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+fma4 -fp-contract=fast | FileCheck %s --check-prefix=FMA4 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq -fp-contract=fast | FileCheck %s --check-prefix=AVX512 + +; +; Pattern: (fadd (fmul x, y), z) -> (fmadd x,y,z) +; + +define <16 x float> @test_16f32_fmadd(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) { +; FMA-LABEL: test_16f32_fmadd: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213ps %ymm4, %ymm2, %ymm0 +; FMA-NEXT: vfmadd213ps %ymm5, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_16f32_fmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddps %ymm4, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfmaddps %ymm5, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_16f32_fmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213ps %zmm2, %zmm1, %zmm0 +; AVX512-NEXT: retq %x = fmul <16 x float> %a0, %a1 %res = fadd <16 x float> %x, %a2 ret <16 x float> %res } -; CHECK-LABEL: test_x86_fmsub_ps_y_wide -; CHECK: vfmsub213ps -; CHECK: vfmsub213ps -; CHECK: ret -; CHECK_FMA4-LABEL: test_x86_fmsub_ps_y_wide -; CHECK_FMA4: vfmsubps -; CHECK_FMA4: vfmsubps -; CHECK_FMA4: ret -define <16 x float> @test_x86_fmsub_ps_y_wide(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) { +define <8 x double> @test_8f64_fmadd(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) { +; FMA-LABEL: test_8f64_fmadd: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213pd %ymm4, %ymm2, %ymm0 +; FMA-NEXT: vfmadd213pd %ymm5, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_8f64_fmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddpd %ymm4, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfmaddpd %ymm5, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_8f64_fmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213pd %zmm2, %zmm1, %zmm0 +; AVX512-NEXT: retq + %x = fmul <8 x double> %a0, %a1 + %res = fadd <8 x double> %x, %a2 + ret <8 x double> %res +} + +; +; Pattern: (fsub (fmul x, y), z) -> (fmsub x, y, z) +; + +define <16 x float> @test_16f32_fmsub(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) { +; FMA-LABEL: test_16f32_fmsub: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213ps %ymm4, %ymm2, %ymm0 +; FMA-NEXT: vfmsub213ps %ymm5, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_16f32_fmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubps %ymm4, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfmsubps %ymm5, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_16f32_fmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213ps %zmm2, %zmm1, %zmm0 +; AVX512-NEXT: retq %x = fmul <16 x float> %a0, %a1 %res = fsub <16 x float> %x, %a2 ret <16 x float> %res } -; CHECK-LABEL: test_x86_fnmadd_ps_y_wide -; CHECK: vfnmadd213ps -; CHECK: vfnmadd213ps -; CHECK: ret -; CHECK_FMA4-LABEL: test_x86_fnmadd_ps_y_wide -; CHECK_FMA4: vfnmaddps -; CHECK_FMA4: vfnmaddps -; CHECK_FMA4: ret -define <16 x float> @test_x86_fnmadd_ps_y_wide(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) { +define <8 x double> @test_8f64_fmsub(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) { +; FMA-LABEL: test_8f64_fmsub: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213pd %ymm4, %ymm2, %ymm0 +; FMA-NEXT: vfmsub213pd %ymm5, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_8f64_fmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubpd %ymm4, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfmsubpd %ymm5, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_8f64_fmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213pd %zmm2, %zmm1, %zmm0 +; AVX512-NEXT: retq + %x = fmul <8 x double> %a0, %a1 + %res = fsub <8 x double> %x, %a2 + ret <8 x double> %res +} + +; +; Pattern: (fsub z, (fmul x, y)) -> (fnmadd x, y, z) +; + +define <16 x float> @test_16f32_fnmadd(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) { +; FMA-LABEL: test_16f32_fnmadd: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213ps %ymm4, %ymm2, %ymm0 +; FMA-NEXT: vfnmadd213ps %ymm5, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_16f32_fnmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddps %ymm4, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfnmaddps %ymm5, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_16f32_fnmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmadd213ps %zmm2, %zmm1, %zmm0 +; AVX512-NEXT: retq %x = fmul <16 x float> %a0, %a1 %res = fsub <16 x float> %a2, %x ret <16 x float> %res } -; CHECK-LABEL: test_x86_fnmsub_ps_y_wide -; CHECK: vfnmsub213ps -; CHECK: vfnmsub213ps -; CHECK: ret -define <16 x float> @test_x86_fnmsub_ps_y_wide(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) { +define <8 x double> @test_8f64_fnmadd(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) { +; FMA-LABEL: test_8f64_fnmadd: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213pd %ymm4, %ymm2, %ymm0 +; FMA-NEXT: vfnmadd213pd %ymm5, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_8f64_fnmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddpd %ymm4, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfnmaddpd %ymm5, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_8f64_fnmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmadd213pd %zmm2, %zmm1, %zmm0 +; AVX512-NEXT: retq + %x = fmul <8 x double> %a0, %a1 + %res = fsub <8 x double> %a2, %x + ret <8 x double> %res +} + +; +; Pattern: (fsub (fneg (fmul x, y)), z) -> (fnmsub x, y, z) +; + +define <16 x float> @test_16f32_fnmsub(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) { +; FMA-LABEL: test_16f32_fnmsub: +; FMA: # BB#0: +; FMA-NEXT: vfnmsub213ps %ymm4, %ymm2, %ymm0 +; FMA-NEXT: vfnmsub213ps %ymm5, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_16f32_fnmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmsubps %ymm4, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfnmsubps %ymm5, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_16f32_fnmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmsub213ps %zmm2, %zmm1, %zmm0 +; AVX512-NEXT: retq %x = fmul <16 x float> %a0, %a1 %y = fsub <16 x float> , %x %res = fsub <16 x float> %y, %a2 ret <16 x float> %res } -; CHECK-LABEL: test_x86_fmadd_pd_y_wide -; CHECK: vfmadd213pd -; CHECK: vfmadd213pd -; CHECK: ret -; CHECK_FMA4-LABEL: test_x86_fmadd_pd_y_wide -; CHECK_FMA4: vfmaddpd -; CHECK_FMA4: vfmaddpd -; CHECK_FMA4: ret -define <8 x double> @test_x86_fmadd_pd_y_wide(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) { +define <8 x double> @test_8f64_fnmsub(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) { +; FMA-LABEL: test_8f64_fnmsub: +; FMA: # BB#0: +; FMA-NEXT: vfnmsub213pd %ymm4, %ymm2, %ymm0 +; FMA-NEXT: vfnmsub213pd %ymm5, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_8f64_fnmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmsubpd %ymm4, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfnmsubpd %ymm5, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_8f64_fnmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmsub213pd %zmm2, %zmm1, %zmm0 +; AVX512-NEXT: retq %x = fmul <8 x double> %a0, %a1 - %res = fadd <8 x double> %x, %a2 + %y = fsub <8 x double> , %x + %res = fsub <8 x double> %y, %a2 ret <8 x double> %res } -; CHECK-LABEL: test_x86_fmsub_pd_y_wide -; CHECK: vfmsub213pd -; CHECK: vfmsub213pd -; CHECK: ret -; CHECK_FMA4-LABEL: test_x86_fmsub_pd_y_wide -; CHECK_FMA4: vfmsubpd -; CHECK_FMA4: vfmsubpd -; CHECK_FMA4: ret -define <8 x double> @test_x86_fmsub_pd_y_wide(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) { - %x = fmul <8 x double> %a0, %a1 - %res = fsub <8 x double> %x, %a2 +; +; Load Folding Patterns +; + +define <16 x float> @test_16f32_fmadd_load(<16 x float>* %a0, <16 x float> %a1, <16 x float> %a2) { +; FMA-LABEL: test_16f32_fmadd_load: +; FMA: # BB#0: +; FMA-NEXT: vfmadd132ps (%rdi), %ymm2, %ymm0 +; FMA-NEXT: vfmadd132ps 32(%rdi), %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_16f32_fmadd_load: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddps %ymm2, (%rdi), %ymm0, %ymm0 +; FMA4-NEXT: vfmaddps %ymm3, 32(%rdi), %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_16f32_fmadd_load: +; AVX512: # BB#0: +; AVX512-NEXT: vmovaps (%rdi), %zmm2 +; AVX512-NEXT: vfmadd213ps %zmm1, %zmm0, %zmm2 +; AVX512-NEXT: vmovaps %zmm2, %zmm0 +; AVX512-NEXT: retq + %x = load <16 x float>, <16 x float>* %a0 + %y = fmul <16 x float> %x, %a1 + %res = fadd <16 x float> %y, %a2 + ret <16 x float> %res +} + +define <8 x double> @test_8f64_fmsub_load(<8 x double>* %a0, <8 x double> %a1, <8 x double> %a2) { +; FMA-LABEL: test_8f64_fmsub_load: +; FMA: # BB#0: +; FMA-NEXT: vfmsub132pd (%rdi), %ymm2, %ymm0 +; FMA-NEXT: vfmsub132pd 32(%rdi), %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_8f64_fmsub_load: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubpd %ymm2, (%rdi), %ymm0, %ymm0 +; FMA4-NEXT: vfmsubpd %ymm3, 32(%rdi), %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_8f64_fmsub_load: +; AVX512: # BB#0: +; AVX512-NEXT: vmovapd (%rdi), %zmm2 +; AVX512-NEXT: vfmsub213pd %zmm1, %zmm0, %zmm2 +; AVX512-NEXT: vmovaps %zmm2, %zmm0 +; AVX512-NEXT: retq + %x = load <8 x double>, <8 x double>* %a0 + %y = fmul <8 x double> %x, %a1 + %res = fsub <8 x double> %y, %a2 ret <8 x double> %res } + +; +; Patterns (+ fneg variants): mul(add(1.0,x),y), mul(sub(1.0,x),y), mul(sub(x,1.0),y) +; + +define <16 x float> @test_v16f32_mul_add_x_one_y(<16 x float> %x, <16 x float> %y) { +; FMA-LABEL: test_v16f32_mul_add_x_one_y: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213ps %ymm2, %ymm2, %ymm0 +; FMA-NEXT: vfmadd213ps %ymm3, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v16f32_mul_add_x_one_y: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddps %ymm2, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfmaddps %ymm3, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v16f32_mul_add_x_one_y: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213ps %zmm1, %zmm1, %zmm0 +; AVX512-NEXT: retq + %a = fadd <16 x float> %x, + %m = fmul <16 x float> %a, %y + ret <16 x float> %m +} + +define <8 x double> @test_v8f64_mul_y_add_x_one(<8 x double> %x, <8 x double> %y) { +; FMA-LABEL: test_v8f64_mul_y_add_x_one: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213pd %ymm2, %ymm2, %ymm0 +; FMA-NEXT: vfmadd213pd %ymm3, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v8f64_mul_y_add_x_one: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddpd %ymm2, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfmaddpd %ymm3, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v8f64_mul_y_add_x_one: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213pd %zmm1, %zmm1, %zmm0 +; AVX512-NEXT: retq + %a = fadd <8 x double> %x, + %m = fmul <8 x double> %y, %a + ret <8 x double> %m +} + +define <16 x float> @test_v16f32_mul_add_x_negone_y(<16 x float> %x, <16 x float> %y) { +; FMA-LABEL: test_v16f32_mul_add_x_negone_y: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213ps %ymm2, %ymm2, %ymm0 +; FMA-NEXT: vfmsub213ps %ymm3, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v16f32_mul_add_x_negone_y: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubps %ymm2, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfmsubps %ymm3, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v16f32_mul_add_x_negone_y: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213ps %zmm1, %zmm1, %zmm0 +; AVX512-NEXT: retq + %a = fadd <16 x float> %x, + %m = fmul <16 x float> %a, %y + ret <16 x float> %m +} + +define <8 x double> @test_v8f64_mul_y_add_x_negone(<8 x double> %x, <8 x double> %y) { +; FMA-LABEL: test_v8f64_mul_y_add_x_negone: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213pd %ymm2, %ymm2, %ymm0 +; FMA-NEXT: vfmsub213pd %ymm3, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v8f64_mul_y_add_x_negone: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubpd %ymm2, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfmsubpd %ymm3, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v8f64_mul_y_add_x_negone: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213pd %zmm1, %zmm1, %zmm0 +; AVX512-NEXT: retq + %a = fadd <8 x double> %x, + %m = fmul <8 x double> %y, %a + ret <8 x double> %m +} + +define <16 x float> @test_v16f32_mul_sub_one_x_y(<16 x float> %x, <16 x float> %y) { +; FMA-LABEL: test_v16f32_mul_sub_one_x_y: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213ps %ymm2, %ymm2, %ymm0 +; FMA-NEXT: vfnmadd213ps %ymm3, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v16f32_mul_sub_one_x_y: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddps %ymm2, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfnmaddps %ymm3, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v16f32_mul_sub_one_x_y: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmadd213ps %zmm1, %zmm1, %zmm0 +; AVX512-NEXT: retq + %s = fsub <16 x float> , %x + %m = fmul <16 x float> %s, %y + ret <16 x float> %m +} + +define <8 x double> @test_v8f64_mul_y_sub_one_x(<8 x double> %x, <8 x double> %y) { +; FMA-LABEL: test_v8f64_mul_y_sub_one_x: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213pd %ymm2, %ymm2, %ymm0 +; FMA-NEXT: vfnmadd213pd %ymm3, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v8f64_mul_y_sub_one_x: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddpd %ymm2, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfnmaddpd %ymm3, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v8f64_mul_y_sub_one_x: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmadd213pd %zmm1, %zmm1, %zmm0 +; AVX512-NEXT: retq + %s = fsub <8 x double> , %x + %m = fmul <8 x double> %y, %s + ret <8 x double> %m +} + +define <16 x float> @test_v16f32_mul_sub_negone_x_y(<16 x float> %x, <16 x float> %y) { +; FMA-LABEL: test_v16f32_mul_sub_negone_x_y: +; FMA: # BB#0: +; FMA-NEXT: vfnmsub213ps %ymm2, %ymm2, %ymm0 +; FMA-NEXT: vfnmsub213ps %ymm3, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v16f32_mul_sub_negone_x_y: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmsubps %ymm2, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfnmsubps %ymm3, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v16f32_mul_sub_negone_x_y: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmsub213ps %zmm1, %zmm1, %zmm0 +; AVX512-NEXT: retq + %s = fsub <16 x float> , %x + %m = fmul <16 x float> %s, %y + ret <16 x float> %m +} + +define <8 x double> @test_v8f64_mul_y_sub_negone_x(<8 x double> %x, <8 x double> %y) { +; FMA-LABEL: test_v8f64_mul_y_sub_negone_x: +; FMA: # BB#0: +; FMA-NEXT: vfnmsub213pd %ymm2, %ymm2, %ymm0 +; FMA-NEXT: vfnmsub213pd %ymm3, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v8f64_mul_y_sub_negone_x: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmsubpd %ymm2, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfnmsubpd %ymm3, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v8f64_mul_y_sub_negone_x: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmsub213pd %zmm1, %zmm1, %zmm0 +; AVX512-NEXT: retq + %s = fsub <8 x double> , %x + %m = fmul <8 x double> %y, %s + ret <8 x double> %m +} + +define <16 x float> @test_v16f32_mul_sub_x_one_y(<16 x float> %x, <16 x float> %y) { +; FMA-LABEL: test_v16f32_mul_sub_x_one_y: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213ps %ymm2, %ymm2, %ymm0 +; FMA-NEXT: vfmsub213ps %ymm3, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v16f32_mul_sub_x_one_y: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubps %ymm2, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfmsubps %ymm3, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v16f32_mul_sub_x_one_y: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213ps %zmm1, %zmm1, %zmm0 +; AVX512-NEXT: retq + %s = fsub <16 x float> %x, + %m = fmul <16 x float> %s, %y + ret <16 x float> %m +} + +define <8 x double> @test_v8f64_mul_y_sub_x_one(<8 x double> %x, <8 x double> %y) { +; FMA-LABEL: test_v8f64_mul_y_sub_x_one: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213pd %ymm2, %ymm2, %ymm0 +; FMA-NEXT: vfmsub213pd %ymm3, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v8f64_mul_y_sub_x_one: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubpd %ymm2, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfmsubpd %ymm3, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v8f64_mul_y_sub_x_one: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213pd %zmm1, %zmm1, %zmm0 +; AVX512-NEXT: retq + %s = fsub <8 x double> %x, + %m = fmul <8 x double> %y, %s + ret <8 x double> %m +} + +define <16 x float> @test_v16f32_mul_sub_x_negone_y(<16 x float> %x, <16 x float> %y) { +; FMA-LABEL: test_v16f32_mul_sub_x_negone_y: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213ps %ymm2, %ymm2, %ymm0 +; FMA-NEXT: vfmadd213ps %ymm3, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v16f32_mul_sub_x_negone_y: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddps %ymm2, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfmaddps %ymm3, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v16f32_mul_sub_x_negone_y: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213ps %zmm1, %zmm1, %zmm0 +; AVX512-NEXT: retq + %s = fsub <16 x float> %x, + %m = fmul <16 x float> %s, %y + ret <16 x float> %m +} + +define <8 x double> @test_v8f64_mul_y_sub_x_negone(<8 x double> %x, <8 x double> %y) { +; FMA-LABEL: test_v8f64_mul_y_sub_x_negone: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213pd %ymm2, %ymm2, %ymm0 +; FMA-NEXT: vfmadd213pd %ymm3, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v8f64_mul_y_sub_x_negone: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddpd %ymm2, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfmaddpd %ymm3, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v8f64_mul_y_sub_x_negone: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213pd %zmm1, %zmm1, %zmm0 +; AVX512-NEXT: retq + %s = fsub <8 x double> %x, + %m = fmul <8 x double> %y, %s + ret <8 x double> %m +} + +; +; Interpolation Patterns: add(mul(x,t),mul(sub(1.0,t),y)) +; + +define <16 x float> @test_v16f32_interp(<16 x float> %x, <16 x float> %y, <16 x float> %t) { +; FMA-LABEL: test_v16f32_interp: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213ps %ymm3, %ymm5, %ymm3 +; FMA-NEXT: vfnmadd213ps %ymm2, %ymm4, %ymm2 +; FMA-NEXT: vfmadd213ps %ymm2, %ymm4, %ymm0 +; FMA-NEXT: vfmadd213ps %ymm3, %ymm5, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v16f32_interp: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddps %ymm3, %ymm3, %ymm5, %ymm3 +; FMA4-NEXT: vfnmaddps %ymm2, %ymm2, %ymm4, %ymm2 +; FMA4-NEXT: vfmaddps %ymm2, %ymm4, %ymm0, %ymm0 +; FMA4-NEXT: vfmaddps %ymm3, %ymm5, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v16f32_interp: +; AVX512: # BB#0: +; AVX512-NEXT: vmovaps %zmm2, %zmm3 +; AVX512-NEXT: vfnmadd213ps %zmm1, %zmm1, %zmm3 +; AVX512-NEXT: vfmadd213ps %zmm3, %zmm2, %zmm0 +; AVX512-NEXT: retq + %t1 = fsub <16 x float> , %t + %tx = fmul <16 x float> %x, %t + %ty = fmul <16 x float> %y, %t1 + %r = fadd <16 x float> %tx, %ty + ret <16 x float> %r +} + +define <8 x double> @test_v8f64_interp(<8 x double> %x, <8 x double> %y, <8 x double> %t) { +; FMA-LABEL: test_v8f64_interp: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213pd %ymm3, %ymm5, %ymm3 +; FMA-NEXT: vfnmadd213pd %ymm2, %ymm4, %ymm2 +; FMA-NEXT: vfmadd213pd %ymm2, %ymm4, %ymm0 +; FMA-NEXT: vfmadd213pd %ymm3, %ymm5, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v8f64_interp: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddpd %ymm3, %ymm3, %ymm5, %ymm3 +; FMA4-NEXT: vfnmaddpd %ymm2, %ymm2, %ymm4, %ymm2 +; FMA4-NEXT: vfmaddpd %ymm2, %ymm4, %ymm0, %ymm0 +; FMA4-NEXT: vfmaddpd %ymm3, %ymm5, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v8f64_interp: +; AVX512: # BB#0: +; AVX512-NEXT: vmovaps %zmm2, %zmm3 +; AVX512-NEXT: vfnmadd213pd %zmm1, %zmm1, %zmm3 +; AVX512-NEXT: vfmadd213pd %zmm3, %zmm2, %zmm0 +; AVX512-NEXT: retq + %t1 = fsub <8 x double> , %t + %tx = fmul <8 x double> %x, %t + %ty = fmul <8 x double> %y, %t1 + %r = fadd <8 x double> %tx, %ty + ret <8 x double> %r +} + +; +; Pattern: (fneg (fma x, y, z)) -> (fma x, -y, -z) +; + +define <16 x float> @test_v16f32_fneg_fmadd(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) #0 { +; FMA-LABEL: test_v16f32_fneg_fmadd: +; FMA: # BB#0: +; FMA-NEXT: vfnmsub213ps %ymm4, %ymm2, %ymm0 +; FMA-NEXT: vfnmsub213ps %ymm5, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v16f32_fneg_fmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmsubps %ymm4, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfnmsubps %ymm5, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v16f32_fneg_fmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmsub213ps %zmm2, %zmm1, %zmm0 +; AVX512-NEXT: retq + %mul = fmul <16 x float> %a0, %a1 + %add = fadd <16 x float> %mul, %a2 + %neg = fsub <16 x float> , %add + ret <16 x float> %neg +} + +define <8 x double> @test_v8f64_fneg_fmsub(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) #0 { +; FMA-LABEL: test_v8f64_fneg_fmsub: +; FMA: # BB#0: +; FMA-NEXT: vfnmadd213pd %ymm4, %ymm2, %ymm0 +; FMA-NEXT: vfnmadd213pd %ymm5, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v8f64_fneg_fmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfnmaddpd %ymm4, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfnmaddpd %ymm5, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v8f64_fneg_fmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfnmadd213pd %zmm2, %zmm1, %zmm0 +; AVX512-NEXT: retq + %mul = fmul <8 x double> %a0, %a1 + %sub = fsub <8 x double> %mul, %a2 + %neg = fsub <8 x double> , %sub + ret <8 x double> %neg +} + +define <16 x float> @test_v16f32_fneg_fnmadd(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) #0 { +; FMA-LABEL: test_v16f32_fneg_fnmadd: +; FMA: # BB#0: +; FMA-NEXT: vfmsub213ps %ymm4, %ymm2, %ymm0 +; FMA-NEXT: vfmsub213ps %ymm5, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v16f32_fneg_fnmadd: +; FMA4: # BB#0: +; FMA4-NEXT: vfmsubps %ymm4, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfmsubps %ymm5, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v16f32_fneg_fnmadd: +; AVX512: # BB#0: +; AVX512-NEXT: vfmsub213ps %zmm2, %zmm1, %zmm0 +; AVX512-NEXT: retq + %mul = fmul <16 x float> %a0, %a1 + %neg0 = fsub <16 x float> , %mul + %add = fadd <16 x float> %neg0, %a2 + %neg1 = fsub <16 x float> , %add + ret <16 x float> %neg1 +} + +define <8 x double> @test_v8f64_fneg_fnmsub(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) #0 { +; FMA-LABEL: test_v8f64_fneg_fnmsub: +; FMA: # BB#0: +; FMA-NEXT: vfmadd213pd %ymm4, %ymm2, %ymm0 +; FMA-NEXT: vfmadd213pd %ymm5, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v8f64_fneg_fnmsub: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddpd %ymm4, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfmaddpd %ymm5, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v8f64_fneg_fnmsub: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd213pd %zmm2, %zmm1, %zmm0 +; AVX512-NEXT: retq + %mul = fmul <8 x double> %a0, %a1 + %neg0 = fsub <8 x double> , %mul + %sub = fsub <8 x double> %neg0, %a2 + %neg1 = fsub <8 x double> , %sub + ret <8 x double> %neg1 +} + +; +; Pattern: (fma x, c1, (fmul x, c2)) -> (fmul x, c1+c2) +; + +define <16 x float> @test_v16f32_fma_x_c1_fmul_x_c2(<16 x float> %x) #0 { +; FMA-LABEL: test_v16f32_fma_x_c1_fmul_x_c2: +; FMA: # BB#0: +; FMA-NEXT: vmulps {{.*}}(%rip), %ymm0, %ymm0 +; FMA-NEXT: vmulps {{.*}}(%rip), %ymm1, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v16f32_fma_x_c1_fmul_x_c2: +; FMA4: # BB#0: +; FMA4-NEXT: vmulps {{.*}}(%rip), %ymm0, %ymm0 +; FMA4-NEXT: vmulps {{.*}}(%rip), %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v16f32_fma_x_c1_fmul_x_c2: +; AVX512: # BB#0: +; AVX512-NEXT: vmulps {{.*}}(%rip), %zmm0, %zmm0 +; AVX512-NEXT: retq + %m0 = fmul <16 x float> %x, + %m1 = fmul <16 x float> %x, + %a = fadd <16 x float> %m0, %m1 + ret <16 x float> %a +} + +; +; Pattern: (fma (fmul x, c1), c2, y) -> (fma x, c1*c2, y) +; + +define <16 x float> @test_v16f32_fma_fmul_x_c1_c2_y(<16 x float> %x, <16 x float> %y) #0 { +; FMA-LABEL: test_v16f32_fma_fmul_x_c1_c2_y: +; FMA: # BB#0: +; FMA-NEXT: vfmadd132ps {{.*}}(%rip), %ymm2, %ymm0 +; FMA-NEXT: vfmadd132ps {{.*}}(%rip), %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v16f32_fma_fmul_x_c1_c2_y: +; FMA4: # BB#0: +; FMA4-NEXT: vfmaddps %ymm2, {{.*}}(%rip), %ymm0, %ymm0 +; FMA4-NEXT: vfmaddps %ymm3, {{.*}}(%rip), %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v16f32_fma_fmul_x_c1_c2_y: +; AVX512: # BB#0: +; AVX512-NEXT: vfmadd231ps {{.*}}(%rip), %zmm0, %zmm1 +; AVX512-NEXT: vmovaps %zmm1, %zmm0 +; AVX512-NEXT: retq + %m0 = fmul <16 x float> %x, + %m1 = fmul <16 x float> %m0, + %a = fadd <16 x float> %m1, %y + ret <16 x float> %a +} + +; Pattern: (fneg (fmul x, y)) -> (fnmsub x, y, 0) + +define <16 x float> @test_v16f32_fneg_fmul(<16 x float> %x, <16 x float> %y) #0 { +; FMA-LABEL: test_v16f32_fneg_fmul: +; FMA: # BB#0: +; FMA-NEXT: vxorps %ymm4, %ymm4, %ymm4 +; FMA-NEXT: vfnmsub213ps %ymm4, %ymm2, %ymm0 +; FMA-NEXT: vfnmsub213ps %ymm4, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v16f32_fneg_fmul: +; FMA4: # BB#0: +; FMA4-NEXT: vxorps %ymm4, %ymm4, %ymm4 +; FMA4-NEXT: vfnmsubps %ymm4, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfnmsubps %ymm4, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v16f32_fneg_fmul: +; AVX512: # BB#0: +; AVX512-NEXT: vpxord %zmm2, %zmm2, %zmm2 +; AVX512-NEXT: vfnmsub213ps %zmm2, %zmm1, %zmm0 +; AVX512-NEXT: retq + %m = fmul nsz <16 x float> %x, %y + %n = fsub <16 x float> , %m + ret <16 x float> %n +} + +define <8 x double> @test_v8f64_fneg_fmul(<8 x double> %x, <8 x double> %y) #0 { +; FMA-LABEL: test_v8f64_fneg_fmul: +; FMA: # BB#0: +; FMA-NEXT: vxorpd %ymm4, %ymm4, %ymm4 +; FMA-NEXT: vfnmsub213pd %ymm4, %ymm2, %ymm0 +; FMA-NEXT: vfnmsub213pd %ymm4, %ymm3, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v8f64_fneg_fmul: +; FMA4: # BB#0: +; FMA4-NEXT: vxorpd %ymm4, %ymm4, %ymm4 +; FMA4-NEXT: vfnmsubpd %ymm4, %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vfnmsubpd %ymm4, %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v8f64_fneg_fmul: +; AVX512: # BB#0: +; AVX512-NEXT: vpxord %zmm2, %zmm2, %zmm2 +; AVX512-NEXT: vfnmsub213pd %zmm2, %zmm1, %zmm0 +; AVX512-NEXT: retq + %m = fmul nsz <8 x double> %x, %y + %n = fsub <8 x double> , %m + ret <8 x double> %n +} + +define <8 x double> @test_v8f64_fneg_fmul_no_nsz(<8 x double> %x, <8 x double> %y) #0 { +; FMA-LABEL: test_v8f64_fneg_fmul_no_nsz: +; FMA: # BB#0: +; FMA-NEXT: vmulpd %ymm3, %ymm1, %ymm1 +; FMA-NEXT: vmulpd %ymm2, %ymm0, %ymm0 +; FMA-NEXT: vmovapd {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808] +; FMA-NEXT: vxorpd %ymm2, %ymm0, %ymm0 +; FMA-NEXT: vxorpd %ymm2, %ymm1, %ymm1 +; FMA-NEXT: retq +; +; FMA4-LABEL: test_v8f64_fneg_fmul_no_nsz: +; FMA4: # BB#0: +; FMA4-NEXT: vmulpd %ymm3, %ymm1, %ymm1 +; FMA4-NEXT: vmulpd %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vmovapd {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808] +; FMA4-NEXT: vxorpd %ymm2, %ymm0, %ymm0 +; FMA4-NEXT: vxorpd %ymm2, %ymm1, %ymm1 +; FMA4-NEXT: retq +; +; AVX512-LABEL: test_v8f64_fneg_fmul_no_nsz: +; AVX512: # BB#0: +; AVX512-NEXT: vmulpd %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: vxorpd {{.*}}(%rip), %zmm0, %zmm0 +; AVX512-NEXT: retq + %m = fmul <8 x double> %x, %y + %n = fsub <8 x double> , %m + ret <8 x double> %n +} + +attributes #0 = { "unsafe-fp-math"="true" } diff --git a/test/CodeGen/X86/fmaxnum.ll b/test/CodeGen/X86/fmaxnum.ll index 23678c46dba0..ebfbd064572a 100644 --- a/test/CodeGen/X86/fmaxnum.ll +++ b/test/CodeGen/X86/fmaxnum.ll @@ -1,4 +1,5 @@ -; RUN: llc -march=x86 -mtriple=i386-linux-gnu < %s | FileCheck %s +; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=sse2 < %s | FileCheck %s --check-prefix=CHECK --check-prefix=SSE +; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=avx < %s | FileCheck %s --check-prefix=CHECK --check-prefix=AVX declare float @fmaxf(float, float) declare double @fmax(double, double) @@ -7,44 +8,232 @@ declare float @llvm.maxnum.f32(float, float) declare double @llvm.maxnum.f64(double, double) declare x86_fp80 @llvm.maxnum.f80(x86_fp80, x86_fp80) +declare <2 x float> @llvm.maxnum.v2f32(<2 x float>, <2 x float>) +declare <4 x float> @llvm.maxnum.v4f32(<4 x float>, <4 x float>) +declare <2 x double> @llvm.maxnum.v2f64(<2 x double>, <2 x double>) +declare <4 x double> @llvm.maxnum.v4f64(<4 x double>, <4 x double>) +declare <8 x double> @llvm.maxnum.v8f64(<8 x double>, <8 x double>) + +; FIXME: As the vector tests show, the SSE run shouldn't need this many moves. + ; CHECK-LABEL: @test_fmaxf -; CHECK: calll fmaxf +; SSE: movaps %xmm0, %xmm2 +; SSE-NEXT: cmpunordss %xmm2, %xmm2 +; SSE-NEXT: movaps %xmm2, %xmm3 +; SSE-NEXT: andps %xmm1, %xmm3 +; SSE-NEXT: maxss %xmm0, %xmm1 +; SSE-NEXT: andnps %xmm1, %xmm2 +; SSE-NEXT: orps %xmm3, %xmm2 +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX: vmaxss %xmm0, %xmm1, %xmm2 +; AVX-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 +; AVX-NEXT: retq define float @test_fmaxf(float %x, float %y) { %z = call float @fmaxf(float %x, float %y) readnone ret float %z } +; CHECK-LABEL: @test_fmaxf_minsize +; CHECK: jmp fmaxf +define float @test_fmaxf_minsize(float %x, float %y) minsize { + %z = call float @fmaxf(float %x, float %y) readnone + ret float %z +} + +; FIXME: As the vector tests show, the SSE run shouldn't need this many moves. + ; CHECK-LABEL: @test_fmax -; CHECK: calll fmax +; SSE: movapd %xmm0, %xmm2 +; SSE-NEXT: cmpunordsd %xmm2, %xmm2 +; SSE-NEXT: movapd %xmm2, %xmm3 +; SSE-NEXT: andpd %xmm1, %xmm3 +; SSE-NEXT: maxsd %xmm0, %xmm1 +; SSE-NEXT: andnpd %xmm1, %xmm2 +; SSE-NEXT: orpd %xmm3, %xmm2 +; SSE-NEXT: movapd %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX: vmaxsd %xmm0, %xmm1, %xmm2 +; AVX-NEXT: vcmpunordsd %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vblendvpd %xmm0, %xmm1, %xmm2, %xmm0 +; AVX-NEXT: retq define double @test_fmax(double %x, double %y) { %z = call double @fmax(double %x, double %y) readnone ret double %z } ; CHECK-LABEL: @test_fmaxl -; CHECK: calll fmaxl +; CHECK: callq fmaxl define x86_fp80 @test_fmaxl(x86_fp80 %x, x86_fp80 %y) { %z = call x86_fp80 @fmaxl(x86_fp80 %x, x86_fp80 %y) readnone ret x86_fp80 %z } ; CHECK-LABEL: @test_intrinsic_fmaxf -; CHECK: calll fmaxf +; SSE: movaps %xmm0, %xmm2 +; SSE-NEXT: cmpunordss %xmm2, %xmm2 +; SSE-NEXT: movaps %xmm2, %xmm3 +; SSE-NEXT: andps %xmm1, %xmm3 +; SSE-NEXT: maxss %xmm0, %xmm1 +; SSE-NEXT: andnps %xmm1, %xmm2 +; SSE-NEXT: orps %xmm3, %xmm2 +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX: vmaxss %xmm0, %xmm1, %xmm2 +; AVX-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 +; AVX-NEXT: retq define float @test_intrinsic_fmaxf(float %x, float %y) { %z = call float @llvm.maxnum.f32(float %x, float %y) readnone ret float %z } + ; CHECK-LABEL: @test_intrinsic_fmax -; CHECK: calll fmax +; SSE: movapd %xmm0, %xmm2 +; SSE-NEXT: cmpunordsd %xmm2, %xmm2 +; SSE-NEXT: movapd %xmm2, %xmm3 +; SSE-NEXT: andpd %xmm1, %xmm3 +; SSE-NEXT: maxsd %xmm0, %xmm1 +; SSE-NEXT: andnpd %xmm1, %xmm2 +; SSE-NEXT: orpd %xmm3, %xmm2 +; SSE-NEXT: movapd %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX: vmaxsd %xmm0, %xmm1, %xmm2 +; AVX-NEXT: vcmpunordsd %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vblendvpd %xmm0, %xmm1, %xmm2, %xmm0 +; AVX-NEXT: retq define double @test_intrinsic_fmax(double %x, double %y) { %z = call double @llvm.maxnum.f64(double %x, double %y) readnone ret double %z } ; CHECK-LABEL: @test_intrinsic_fmaxl -; CHECK: calll fmaxl +; CHECK: callq fmaxl define x86_fp80 @test_intrinsic_fmaxl(x86_fp80 %x, x86_fp80 %y) { %z = call x86_fp80 @llvm.maxnum.f80(x86_fp80 %x, x86_fp80 %y) readnone ret x86_fp80 %z } + +; CHECK-LABEL: @test_intrinsic_fmax_v2f32 +; SSE: movaps %xmm1, %xmm2 +; SSE-NEXT: maxps %xmm0, %xmm2 +; SSE-NEXT: cmpunordps %xmm0, %xmm0 +; SSE-NEXT: andps %xmm0, %xmm1 +; SSE-NEXT: andnps %xmm2, %xmm0 +; SSE-NEXT: orps %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX: vmaxps %xmm0, %xmm1, %xmm2 +; AVX-NEXT: vcmpunordps %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 +; AVX-NEXT: retq +define <2 x float> @test_intrinsic_fmax_v2f32(<2 x float> %x, <2 x float> %y) { + %z = call <2 x float> @llvm.maxnum.v2f32(<2 x float> %x, <2 x float> %y) readnone + ret <2 x float> %z +} + +; CHECK-LABEL: @test_intrinsic_fmax_v4f32 +; SSE: movaps %xmm1, %xmm2 +; SSE-NEXT: maxps %xmm0, %xmm2 +; SSE-NEXT: cmpunordps %xmm0, %xmm0 +; SSE-NEXT: andps %xmm0, %xmm1 +; SSE-NEXT: andnps %xmm2, %xmm0 +; SSE-NEXT: orps %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX: vmaxps %xmm0, %xmm1, %xmm2 +; AVX-NEXT: vcmpunordps %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 +; AVX-NEXT: retq +define <4 x float> @test_intrinsic_fmax_v4f32(<4 x float> %x, <4 x float> %y) { + %z = call <4 x float> @llvm.maxnum.v4f32(<4 x float> %x, <4 x float> %y) readnone + ret <4 x float> %z +} + +; CHECK-LABEL: @test_intrinsic_fmax_v2f64 +; SSE: movapd %xmm1, %xmm2 +; SSE-NEXT: maxpd %xmm0, %xmm2 +; SSE-NEXT: cmpunordpd %xmm0, %xmm0 +; SSE-NEXT: andpd %xmm0, %xmm1 +; SSE-NEXT: andnpd %xmm2, %xmm0 +; SSE-NEXT: orpd %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX: vmaxpd %xmm0, %xmm1, %xmm2 +; AVX-NEXT: vcmpunordpd %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vblendvpd %xmm0, %xmm1, %xmm2, %xmm0 +; AVX-NEXT: retq +define <2 x double> @test_intrinsic_fmax_v2f64(<2 x double> %x, <2 x double> %y) { + %z = call <2 x double> @llvm.maxnum.v2f64(<2 x double> %x, <2 x double> %y) readnone + ret <2 x double> %z +} + +; CHECK-LABEL: @test_intrinsic_fmax_v4f64 +; SSE: movapd %xmm2, %xmm4 +; SSE-NEXT: maxpd %xmm0, %xmm4 +; SSE-NEXT: cmpunordpd %xmm0, %xmm0 +; SSE-NEXT: andpd %xmm0, %xmm2 +; SSE-NEXT: andnpd %xmm4, %xmm0 +; SSE-NEXT: orpd %xmm2, %xmm0 +; SSE-NEXT: movapd %xmm3, %xmm2 +; SSE-NEXT: maxpd %xmm1, %xmm2 +; SSE-NEXT: cmpunordpd %xmm1, %xmm1 +; SSE-NEXT: andpd %xmm1, %xmm3 +; SSE-NEXT: andnpd %xmm2, %xmm1 +; SSE-NEXT: orpd %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX: vmaxpd %ymm0, %ymm1, %ymm2 +; AVX-NEXT: vcmpunordpd %ymm0, %ymm0, %ymm0 +; AVX-NEXT: vblendvpd %ymm0, %ymm1, %ymm2, %ymm0 +; AVX-NEXT: retq +define <4 x double> @test_intrinsic_fmax_v4f64(<4 x double> %x, <4 x double> %y) { + %z = call <4 x double> @llvm.maxnum.v4f64(<4 x double> %x, <4 x double> %y) readnone + ret <4 x double> %z +} + +; CHECK-LABEL: @test_intrinsic_fmax_v8f64 +; SSE: movapd %xmm4, %xmm8 +; SSE-NEXT: maxpd %xmm0, %xmm8 +; SSE-NEXT: cmpunordpd %xmm0, %xmm0 +; SSE-NEXT: andpd %xmm0, %xmm4 +; SSE-NEXT: andnpd %xmm8, %xmm0 +; SSE-NEXT: orpd %xmm4, %xmm0 +; SSE-NEXT: movapd %xmm5, %xmm4 +; SSE-NEXT: maxpd %xmm1, %xmm4 +; SSE-NEXT: cmpunordpd %xmm1, %xmm1 +; SSE-NEXT: andpd %xmm1, %xmm5 +; SSE-NEXT: andnpd %xmm4, %xmm1 +; SSE-NEXT: orpd %xmm5, %xmm1 +; SSE-NEXT: movapd %xmm6, %xmm4 +; SSE-NEXT: maxpd %xmm2, %xmm4 +; SSE-NEXT: cmpunordpd %xmm2, %xmm2 +; SSE-NEXT: andpd %xmm2, %xmm6 +; SSE-NEXT: andnpd %xmm4, %xmm2 +; SSE-NEXT: orpd %xmm6, %xmm2 +; SSE-NEXT: movapd %xmm7, %xmm4 +; SSE-NEXT: maxpd %xmm3, %xmm4 +; SSE-NEXT: cmpunordpd %xmm3, %xmm3 +; SSE-NEXT: andpd %xmm3, %xmm7 +; SSE-NEXT: andnpd %xmm4, %xmm3 +; SSE-NEXT: orpd %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX: vmaxpd %ymm0, %ymm2, %ymm4 +; AVX-NEXT: vcmpunordpd %ymm0, %ymm0, %ymm0 +; AVX-NEXT: vblendvpd %ymm0, %ymm2, %ymm4, %ymm0 +; AVX-NEXT: vmaxpd %ymm1, %ymm3, %ymm2 +; AVX-NEXT: vcmpunordpd %ymm1, %ymm1, %ymm1 +; AVX-NEXT: vblendvpd %ymm1, %ymm3, %ymm2, %ymm1 +; AVX-NEXT: retq +define <8 x double> @test_intrinsic_fmax_v8f64(<8 x double> %x, <8 x double> %y) { + %z = call <8 x double> @llvm.maxnum.v8f64(<8 x double> %x, <8 x double> %y) readnone + ret <8 x double> %z +} + diff --git a/test/CodeGen/X86/fminnum.ll b/test/CodeGen/X86/fminnum.ll index 1e33cf4696af..afe8b804f267 100644 --- a/test/CodeGen/X86/fminnum.ll +++ b/test/CodeGen/X86/fminnum.ll @@ -1,4 +1,5 @@ -; RUN: llc -march=x86 -mtriple=i386-linux-gnu -mattr=+sse,+sse2 < %s | FileCheck %s +; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=sse2 < %s | FileCheck %s --check-prefix=CHECK --check-prefix=SSE +; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=avx < %s | FileCheck %s --check-prefix=CHECK --check-prefix=AVX declare float @fminf(float, float) declare double @fmin(double, double) @@ -10,85 +11,219 @@ declare x86_fp80 @llvm.minnum.f80(x86_fp80, x86_fp80) declare <2 x float> @llvm.minnum.v2f32(<2 x float>, <2 x float>) declare <4 x float> @llvm.minnum.v4f32(<4 x float>, <4 x float>) declare <2 x double> @llvm.minnum.v2f64(<2 x double>, <2 x double>) +declare <4 x double> @llvm.minnum.v4f64(<4 x double>, <4 x double>) declare <8 x double> @llvm.minnum.v8f64(<8 x double>, <8 x double>) +; FIXME: As the vector tests show, the SSE run shouldn't need this many moves. + ; CHECK-LABEL: @test_fminf -; CHECK: jmp fminf +; SSE: movaps %xmm0, %xmm2 +; SSE-NEXT: cmpunordss %xmm2, %xmm2 +; SSE-NEXT: movaps %xmm2, %xmm3 +; SSE-NEXT: andps %xmm1, %xmm3 +; SSE-NEXT: minss %xmm0, %xmm1 +; SSE-NEXT: andnps %xmm1, %xmm2 +; SSE-NEXT: orps %xmm3, %xmm2 +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX: vminss %xmm0, %xmm1, %xmm2 +; AVX-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 +; AVX-NEXT: retq define float @test_fminf(float %x, float %y) { %z = call float @fminf(float %x, float %y) readnone ret float %z } +; FIXME: As the vector tests show, the SSE run shouldn't need this many moves. + ; CHECK-LABEL: @test_fmin -; CHECK: jmp fmin +; SSE: movapd %xmm0, %xmm2 +; SSE-NEXT: cmpunordsd %xmm2, %xmm2 +; SSE-NEXT: movapd %xmm2, %xmm3 +; SSE-NEXT: andpd %xmm1, %xmm3 +; SSE-NEXT: minsd %xmm0, %xmm1 +; SSE-NEXT: andnpd %xmm1, %xmm2 +; SSE-NEXT: orpd %xmm3, %xmm2 +; SSE-NEXT: movapd %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX: vminsd %xmm0, %xmm1, %xmm2 +; AVX-NEXT: vcmpunordsd %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vblendvpd %xmm0, %xmm1, %xmm2, %xmm0 +; AVX-NEXT: retq define double @test_fmin(double %x, double %y) { %z = call double @fmin(double %x, double %y) readnone ret double %z } ; CHECK-LABEL: @test_fminl -; CHECK: calll fminl +; CHECK: callq fminl define x86_fp80 @test_fminl(x86_fp80 %x, x86_fp80 %y) { %z = call x86_fp80 @fminl(x86_fp80 %x, x86_fp80 %y) readnone ret x86_fp80 %z } ; CHECK-LABEL: @test_intrinsic_fminf -; CHECK: jmp fminf +; SSE: movaps %xmm0, %xmm2 +; SSE-NEXT: cmpunordss %xmm2, %xmm2 +; SSE-NEXT: movaps %xmm2, %xmm3 +; SSE-NEXT: andps %xmm1, %xmm3 +; SSE-NEXT: minss %xmm0, %xmm1 +; SSE-NEXT: andnps %xmm1, %xmm2 +; SSE-NEXT: orps %xmm3, %xmm2 +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX: vminss %xmm0, %xmm1, %xmm2 +; AVX-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 +; AVX-NEXT: retq define float @test_intrinsic_fminf(float %x, float %y) { %z = call float @llvm.minnum.f32(float %x, float %y) readnone ret float %z } ; CHECK-LABEL: @test_intrinsic_fmin -; CHECK: jmp fmin +; SSE: movapd %xmm0, %xmm2 +; SSE-NEXT: cmpunordsd %xmm2, %xmm2 +; SSE-NEXT: movapd %xmm2, %xmm3 +; SSE-NEXT: andpd %xmm1, %xmm3 +; SSE-NEXT: minsd %xmm0, %xmm1 +; SSE-NEXT: andnpd %xmm1, %xmm2 +; SSE-NEXT: orpd %xmm3, %xmm2 +; SSE-NEXT: movapd %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX: vminsd %xmm0, %xmm1, %xmm2 +; AVX-NEXT: vcmpunordsd %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vblendvpd %xmm0, %xmm1, %xmm2, %xmm0 +; AVX-NEXT: retq define double @test_intrinsic_fmin(double %x, double %y) { %z = call double @llvm.minnum.f64(double %x, double %y) readnone ret double %z } ; CHECK-LABEL: @test_intrinsic_fminl -; CHECK: calll fminl +; CHECK: callq fminl define x86_fp80 @test_intrinsic_fminl(x86_fp80 %x, x86_fp80 %y) { %z = call x86_fp80 @llvm.minnum.f80(x86_fp80 %x, x86_fp80 %y) readnone ret x86_fp80 %z } ; CHECK-LABEL: @test_intrinsic_fmin_v2f32 -; CHECK: calll fminf -; CHECK: calll fminf +; SSE: movaps %xmm1, %xmm2 +; SSE-NEXT: minps %xmm0, %xmm2 +; SSE-NEXT: cmpunordps %xmm0, %xmm0 +; SSE-NEXT: andps %xmm0, %xmm1 +; SSE-NEXT: andnps %xmm2, %xmm0 +; SSE-NEXT: orps %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX: vminps %xmm0, %xmm1, %xmm2 +; AVX-NEXT: vcmpunordps %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 +; AVX-NEXT: retq define <2 x float> @test_intrinsic_fmin_v2f32(<2 x float> %x, <2 x float> %y) { %z = call <2 x float> @llvm.minnum.v2f32(<2 x float> %x, <2 x float> %y) readnone ret <2 x float> %z } ; CHECK-LABEL: @test_intrinsic_fmin_v4f32 -; CHECK: calll fminf -; CHECK: calll fminf -; CHECK: calll fminf -; CHECK: calll fminf +; SSE: movaps %xmm1, %xmm2 +; SSE-NEXT: minps %xmm0, %xmm2 +; SSE-NEXT: cmpunordps %xmm0, %xmm0 +; SSE-NEXT: andps %xmm0, %xmm1 +; SSE-NEXT: andnps %xmm2, %xmm0 +; SSE-NEXT: orps %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX: vminps %xmm0, %xmm1, %xmm2 +; AVX-NEXT: vcmpunordps %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 +; AVX-NEXT: retq define <4 x float> @test_intrinsic_fmin_v4f32(<4 x float> %x, <4 x float> %y) { %z = call <4 x float> @llvm.minnum.v4f32(<4 x float> %x, <4 x float> %y) readnone ret <4 x float> %z } ; CHECK-LABEL: @test_intrinsic_fmin_v2f64 -; CHECK: calll fmin -; CHECK: calll fmin +; SSE: movapd %xmm1, %xmm2 +; SSE-NEXT: minpd %xmm0, %xmm2 +; SSE-NEXT: cmpunordpd %xmm0, %xmm0 +; SSE-NEXT: andpd %xmm0, %xmm1 +; SSE-NEXT: andnpd %xmm2, %xmm0 +; SSE-NEXT: orpd %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX: vminpd %xmm0, %xmm1, %xmm2 +; AVX-NEXT: vcmpunordpd %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vblendvpd %xmm0, %xmm1, %xmm2, %xmm0 +; AVX-NEXT: retq define <2 x double> @test_intrinsic_fmin_v2f64(<2 x double> %x, <2 x double> %y) { %z = call <2 x double> @llvm.minnum.v2f64(<2 x double> %x, <2 x double> %y) readnone ret <2 x double> %z } +; CHECK-LABEL: @test_intrinsic_fmin_v4f64 +; SSE: movapd %xmm2, %xmm4 +; SSE-NEXT: minpd %xmm0, %xmm4 +; SSE-NEXT: cmpunordpd %xmm0, %xmm0 +; SSE-NEXT: andpd %xmm0, %xmm2 +; SSE-NEXT: andnpd %xmm4, %xmm0 +; SSE-NEXT: orpd %xmm2, %xmm0 +; SSE-NEXT: movapd %xmm3, %xmm2 +; SSE-NEXT: minpd %xmm1, %xmm2 +; SSE-NEXT: cmpunordpd %xmm1, %xmm1 +; SSE-NEXT: andpd %xmm1, %xmm3 +; SSE-NEXT: andnpd %xmm2, %xmm1 +; SSE-NEXT: orpd %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX: vminpd %ymm0, %ymm1, %ymm2 +; AVX-NEXT: vcmpunordpd %ymm0, %ymm0, %ymm0 +; AVX-NEXT: vblendvpd %ymm0, %ymm1, %ymm2, %ymm0 +; AVX-NEXT: retq +define <4 x double> @test_intrinsic_fmin_v4f64(<4 x double> %x, <4 x double> %y) { + %z = call <4 x double> @llvm.minnum.v4f64(<4 x double> %x, <4 x double> %y) readnone + ret <4 x double> %z +} + ; CHECK-LABEL: @test_intrinsic_fmin_v8f64 -; CHECK: calll fmin -; CHECK: calll fmin -; CHECK: calll fmin -; CHECK: calll fmin -; CHECK: calll fmin -; CHECK: calll fmin -; CHECK: calll fmin -; CHECK: calll fmin +; SSE: movapd %xmm4, %xmm8 +; SSE-NEXT: minpd %xmm0, %xmm8 +; SSE-NEXT: cmpunordpd %xmm0, %xmm0 +; SSE-NEXT: andpd %xmm0, %xmm4 +; SSE-NEXT: andnpd %xmm8, %xmm0 +; SSE-NEXT: orpd %xmm4, %xmm0 +; SSE-NEXT: movapd %xmm5, %xmm4 +; SSE-NEXT: minpd %xmm1, %xmm4 +; SSE-NEXT: cmpunordpd %xmm1, %xmm1 +; SSE-NEXT: andpd %xmm1, %xmm5 +; SSE-NEXT: andnpd %xmm4, %xmm1 +; SSE-NEXT: orpd %xmm5, %xmm1 +; SSE-NEXT: movapd %xmm6, %xmm4 +; SSE-NEXT: minpd %xmm2, %xmm4 +; SSE-NEXT: cmpunordpd %xmm2, %xmm2 +; SSE-NEXT: andpd %xmm2, %xmm6 +; SSE-NEXT: andnpd %xmm4, %xmm2 +; SSE-NEXT: orpd %xmm6, %xmm2 +; SSE-NEXT: movapd %xmm7, %xmm4 +; SSE-NEXT: minpd %xmm3, %xmm4 +; SSE-NEXT: cmpunordpd %xmm3, %xmm3 +; SSE-NEXT: andpd %xmm3, %xmm7 +; SSE-NEXT: andnpd %xmm4, %xmm3 +; SSE-NEXT: orpd %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX: vminpd %ymm0, %ymm2, %ymm4 +; AVX-NEXT: vcmpunordpd %ymm0, %ymm0, %ymm0 +; AVX-NEXT: vblendvpd %ymm0, %ymm2, %ymm4, %ymm0 +; AVX-NEXT: vminpd %ymm1, %ymm3, %ymm2 +; AVX-NEXT: vcmpunordpd %ymm1, %ymm1, %ymm1 +; AVX-NEXT: vblendvpd %ymm1, %ymm3, %ymm2, %ymm1 +; AVX-NEXT: retq define <8 x double> @test_intrinsic_fmin_v8f64(<8 x double> %x, <8 x double> %y) { %z = call <8 x double> @llvm.minnum.v8f64(<8 x double> %x, <8 x double> %y) readnone ret <8 x double> %z diff --git a/test/CodeGen/X86/fmul-combines.ll b/test/CodeGen/X86/fmul-combines.ll index 7d75611e1330..564ce42fdb75 100644 --- a/test/CodeGen/X86/fmul-combines.ll +++ b/test/CodeGen/X86/fmul-combines.ll @@ -56,10 +56,10 @@ define <4 x float> @fmul_c3_c4_v4f32(<4 x float> %x) #0 { } ; We should be able to pre-multiply the two constant vectors. -; CHECK: float 5.000000e+00 -; CHECK: float 1.200000e+01 -; CHECK: float 2.100000e+01 -; CHECK: float 3.200000e+01 +; CHECK: float 5 +; CHECK: float 12 +; CHECK: float 21 +; CHECK: float 32 ; CHECK-LABEL: fmul_v4f32_two_consts_no_splat: ; CHECK: mulps ; CHECK-NOT: mulps @@ -71,10 +71,10 @@ define <4 x float> @fmul_v4f32_two_consts_no_splat(<4 x float> %x) #0 { } ; Same as above, but reverse operands to make sure non-canonical form is also handled. -; CHECK: float 5.000000e+00 -; CHECK: float 1.200000e+01 -; CHECK: float 2.100000e+01 -; CHECK: float 3.200000e+01 +; CHECK: float 5 +; CHECK: float 12 +; CHECK: float 21 +; CHECK: float 32 ; CHECK-LABEL: fmul_v4f32_two_consts_no_splat_non_canonical: ; CHECK: mulps ; CHECK-NOT: mulps @@ -86,15 +86,13 @@ define <4 x float> @fmul_v4f32_two_consts_no_splat_non_canonical(<4 x float> %x) } ; More than one use of a constant multiply should not inhibit the optimization. -; Instead of a chain of 2 dependent mults, this test will have 2 independent mults. -; CHECK: float 5.000000e+00 -; CHECK: float 1.200000e+01 -; CHECK: float 2.100000e+01 -; CHECK: float 3.200000e+01 +; Instead of a chain of 2 dependent mults, this test will have 2 independent mults. +; CHECK: float 6 +; CHECK: float 14 +; CHECK: float 24 +; CHECK: float 36 ; CHECK-LABEL: fmul_v4f32_two_consts_no_splat_multiple_use: ; CHECK: mulps -; CHECK: mulps -; CHECK: addps ; CHECK: ret define <4 x float> @fmul_v4f32_two_consts_no_splat_multiple_use(<4 x float> %x) #0 { %y = fmul <4 x float> %x, @@ -112,10 +110,10 @@ define <4 x float> @PR22698_splats(<4 x float> %a) #0 { %mul3 = fmul fast <4 x float> %a, %mul2 ret <4 x float> %mul3 -; CHECK: float 2.400000e+01 -; CHECK: float 2.400000e+01 -; CHECK: float 2.400000e+01 -; CHECK: float 2.400000e+01 +; CHECK: float 24 +; CHECK: float 24 +; CHECK: float 24 +; CHECK: float 24 ; CHECK-LABEL: PR22698_splats: ; CHECK: mulps ; CHECK: ret @@ -128,10 +126,10 @@ define <4 x float> @PR22698_no_splats(<4 x float> %a) #0 { %mul3 = fmul fast <4 x float> %a, %mul2 ret <4 x float> %mul3 -; CHECK: float 4.500000e+01 -; CHECK: float 1.200000e+02 -; CHECK: float 2.310000e+02 -; CHECK: float 3.840000e+02 +; CHECK: float 45 +; CHECK: float 120 +; CHECK: float 231 +; CHECK: float 384 ; CHECK-LABEL: PR22698_no_splats: ; CHECK: mulps ; CHECK: ret diff --git a/test/CodeGen/X86/fold-load-binops.ll b/test/CodeGen/X86/fold-load-binops.ll index 6d501c74fe57..43966f60718b 100644 --- a/test/CodeGen/X86/fold-load-binops.ll +++ b/test/CodeGen/X86/fold-load-binops.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+sse2 < %s | FileCheck %s --check-prefix=SSE ; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+avx < %s | FileCheck %s --check-prefix=AVX diff --git a/test/CodeGen/X86/fold-load-unops.ll b/test/CodeGen/X86/fold-load-unops.ll index fcde0218158a..bedda3f297da 100644 --- a/test/CodeGen/X86/fold-load-unops.ll +++ b/test/CodeGen/X86/fold-load-unops.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+sse2 < %s | FileCheck %s --check-prefix=SSE ; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+avx < %s | FileCheck %s --check-prefix=AVX diff --git a/test/CodeGen/X86/fold-push.ll b/test/CodeGen/X86/fold-push.ll new file mode 100644 index 000000000000..eaf91351021f --- /dev/null +++ b/test/CodeGen/X86/fold-push.ll @@ -0,0 +1,40 @@ +; RUN: llc < %s -mtriple=i686-windows | FileCheck %s -check-prefix=CHECK -check-prefix=NORMAL +; RUN: llc < %s -mtriple=i686-windows -mattr=call-reg-indirect | FileCheck %s -check-prefix=CHECK -check-prefix=SLM + +declare void @foo(i32 %r) + +define void @test(i32 %a, i32 %b) optsize nounwind { +; CHECK-LABEL: test: +; CHECK: movl [[EAX:%e..]], (%esp) +; CHECK-NEXT: pushl [[EAX]] +; CHECK-NEXT: calll +; CHECK-NEXT: addl $4, %esp +; CHECK: nop +; NORMAL: pushl (%esp) +; SLM: movl (%esp), [[RELOAD:%e..]] +; SLM-NEXT: pushl [[RELOAD]] +; CHECK: calll +; CHECK-NEXT: addl $4, %esp + %c = add i32 %a, %b + call void @foo(i32 %c) + call void asm sideeffect "nop", "~{ax},~{bx},~{cx},~{dx},~{bp},~{si},~{di}"() + call void @foo(i32 %c) + ret void +} + +define void @test_min(i32 %a, i32 %b) minsize nounwind { +; CHECK-LABEL: test_min: +; CHECK: movl [[EAX:%e..]], (%esp) +; CHECK-NEXT: pushl [[EAX]] +; CHECK-NEXT: calll +; CHECK-NEXT: popl +; CHECK: nop +; CHECK: pushl (%esp) +; CHECK: calll +; CHECK-NEXT: popl + %c = add i32 %a, %b + call void @foo(i32 %c) + call void asm sideeffect "nop", "~{ax},~{bx},~{cx},~{dx},~{bp},~{si},~{di}"() + call void @foo(i32 %c) + ret void +} diff --git a/test/CodeGen/X86/force-align-stack-alloca.ll b/test/CodeGen/X86/force-align-stack-alloca.ll index a9ba20f45e84..d0cf34170081 100644 --- a/test/CodeGen/X86/force-align-stack-alloca.ll +++ b/test/CodeGen/X86/force-align-stack-alloca.ll @@ -3,7 +3,7 @@ ; arbitrarily force alignment up to 32-bytes for i386 hoping that this will ; exceed any ABI provisions. ; -; RUN: llc < %s -mcpu=generic -force-align-stack -stack-alignment=32 | FileCheck %s +; RUN: llc < %s -mcpu=generic -stackrealign -stack-alignment=32 | FileCheck %s target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S128" target triple = "i386-unknown-linux-gnu" diff --git a/test/CodeGen/X86/force-align-stack.ll b/test/CodeGen/X86/force-align-stack.ll index ffcbf8a908c8..fa94ad4dcd86 100644 --- a/test/CodeGen/X86/force-align-stack.ll +++ b/test/CodeGen/X86/force-align-stack.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -relocation-model=static -force-align-stack | FileCheck %s +; RUN: llc < %s -relocation-model=static -stackrealign | FileCheck %s ; Tests to make sure that we always align the stack out to the minimum needed - ; in this case 16-bytes. target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32" diff --git a/test/CodeGen/X86/fp-fast.ll b/test/CodeGen/X86/fp-fast.ll index 27af5738ca3e..fa31b9c9e128 100644 --- a/test/CodeGen/X86/fp-fast.ll +++ b/test/CodeGen/X86/fp-fast.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=avx -enable-unsafe-fp-math < %s | FileCheck %s define float @test1(float %a) { diff --git a/test/CodeGen/X86/fp-logic.ll b/test/CodeGen/X86/fp-logic.ll new file mode 100644 index 000000000000..64c3f6b79a23 --- /dev/null +++ b/test/CodeGen/X86/fp-logic.ll @@ -0,0 +1,264 @@ +; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=sse2 < %s | FileCheck %s + +; PR22428: https://llvm.org/bugs/show_bug.cgi?id=22428 +; f1, f2, f3, and f4 should use an integer logic instruction. +; f9 and f10 should use an FP (SSE) logic instruction. +; +; f5, f6, f7, and f8 are less clear. +; +; For f5 and f6, we can save a register move by using an FP logic instruction, +; but we may need to calculate the relative costs of an SSE op vs. int op vs. +; scalar <-> SSE register moves. +; +; For f7 and f8, the SSE instructions don't take immediate operands, so if we +; use one of those, we either have to load a constant from memory or move the +; scalar immediate value from an integer register over to an SSE register. +; Optimizing for size may affect that decision. Also, note that there are no +; scalar versions of the FP logic ops, so if we want to fold a load into a +; logic op, we have to load or splat a 16-byte vector constant. + +; 1 FP operand, 1 int operand, int result + +define i32 @f1(float %x, i32 %y) { +; CHECK-LABEL: f1: +; CHECK: # BB#0: +; CHECK-NEXT: movd %xmm0, %eax +; CHECK-NEXT: andl %edi, %eax +; CHECK-NEXT: retq + + %bc1 = bitcast float %x to i32 + %and = and i32 %bc1, %y + ret i32 %and +} + +; Swap operands of the logic op. + +define i32 @f2(float %x, i32 %y) { +; CHECK-LABEL: f2: +; CHECK: # BB#0: +; CHECK-NEXT: movd %xmm0, %eax +; CHECK-NEXT: andl %edi, %eax +; CHECK-NEXT: retq + + %bc1 = bitcast float %x to i32 + %and = and i32 %y, %bc1 + ret i32 %and +} + +; 1 FP operand, 1 constant operand, int result + +define i32 @f3(float %x) { +; CHECK-LABEL: f3: +; CHECK: # BB#0: +; CHECK-NEXT: movd %xmm0, %eax +; CHECK-NEXT: andl $1, %eax +; CHECK-NEXT: retq + + %bc1 = bitcast float %x to i32 + %and = and i32 %bc1, 1 + ret i32 %and +} + +; Swap operands of the logic op. + +define i32 @f4(float %x) { +; CHECK-LABEL: f4: +; CHECK: # BB#0: +; CHECK-NEXT: movd %xmm0, %eax +; CHECK-NEXT: andl $2, %eax +; CHECK-NEXT: retq + + %bc1 = bitcast float %x to i32 + %and = and i32 2, %bc1 + ret i32 %and +} + +; 1 FP operand, 1 integer operand, FP result + +define float @f5(float %x, i32 %y) { +; CHECK-LABEL: f5: +; CHECK: # BB#0: +; CHECK-NEXT: movd %xmm0, %eax +; CHECK-NEXT: andl %edi, %eax +; CHECK-NEXT: movd %eax, %xmm0 +; CHECK-NEXT: retq + + %bc1 = bitcast float %x to i32 + %and = and i32 %bc1, %y + %bc2 = bitcast i32 %and to float + ret float %bc2 +} + +; Swap operands of the logic op. + +define float @f6(float %x, i32 %y) { +; CHECK-LABEL: f6: +; CHECK: # BB#0: +; CHECK-NEXT: movd %xmm0, %eax +; CHECK-NEXT: andl %edi, %eax +; CHECK-NEXT: movd %eax, %xmm0 +; CHECK-NEXT: retq + + %bc1 = bitcast float %x to i32 + %and = and i32 %y, %bc1 + %bc2 = bitcast i32 %and to float + ret float %bc2 +} + +; 1 FP operand, 1 constant operand, FP result + +define float @f7(float %x) { +; CHECK-LABEL: f7: +; CHECK: # BB#0: +; CHECK-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero +; CHECK-NEXT: andps %xmm1, %xmm0 +; CHECK-NEXT: retq + + %bc1 = bitcast float %x to i32 + %and = and i32 %bc1, 3 + %bc2 = bitcast i32 %and to float + ret float %bc2 +} + +; Swap operands of the logic op. + +define float @f8(float %x) { +; CHECK-LABEL: f8: +; CHECK: # BB#0: +; CHECK-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero +; CHECK-NEXT: andps %xmm1, %xmm0 +; CHECK-NEXT: retq + + %bc1 = bitcast float %x to i32 + %and = and i32 4, %bc1 + %bc2 = bitcast i32 %and to float + ret float %bc2 +} + +; 2 FP operands, int result + +define i32 @f9(float %x, float %y) { +; CHECK-LABEL: f9: +; CHECK: # BB#0: +; CHECK-NEXT: andps %xmm1, %xmm0 +; CHECK-NEXT: movd %xmm0, %eax +; CHECK-NEXT: retq + + %bc1 = bitcast float %x to i32 + %bc2 = bitcast float %y to i32 + %and = and i32 %bc1, %bc2 + ret i32 %and +} + +; 2 FP operands, FP result + +define float @f10(float %x, float %y) { +; CHECK-LABEL: f10: +; CHECK: # BB#0: +; CHECK-NEXT: andps %xmm1, %xmm0 +; CHECK-NEXT: retq + + %bc1 = bitcast float %x to i32 + %bc2 = bitcast float %y to i32 + %and = and i32 %bc1, %bc2 + %bc3 = bitcast i32 %and to float + ret float %bc3 +} + +define float @or(float %x, float %y) { +; CHECK-LABEL: or: +; CHECK: # BB#0: +; CHECK-NEXT: orps %xmm1, %xmm0 +; CHECK-NEXT: retq + + %bc1 = bitcast float %x to i32 + %bc2 = bitcast float %y to i32 + %and = or i32 %bc1, %bc2 + %bc3 = bitcast i32 %and to float + ret float %bc3 +} + +define float @xor(float %x, float %y) { +; CHECK-LABEL: xor: +; CHECK: # BB#0: +; CHECK-NEXT: xorps %xmm1, %xmm0 +; CHECK-NEXT: retq + + %bc1 = bitcast float %x to i32 + %bc2 = bitcast float %y to i32 + %and = xor i32 %bc1, %bc2 + %bc3 = bitcast i32 %and to float + ret float %bc3 +} + +define float @f7_or(float %x) { +; CHECK-LABEL: f7_or: +; CHECK: # BB#0: +; CHECK-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero +; CHECK-NEXT: orps %xmm1, %xmm0 +; CHECK-NEXT: retq + + %bc1 = bitcast float %x to i32 + %and = or i32 %bc1, 3 + %bc2 = bitcast i32 %and to float + ret float %bc2 +} + +define float @f7_xor(float %x) { +; CHECK-LABEL: f7_xor: +; CHECK: # BB#0: +; CHECK-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero +; CHECK-NEXT: xorps %xmm1, %xmm0 +; CHECK-NEXT: retq + + %bc1 = bitcast float %x to i32 + %and = xor i32 %bc1, 3 + %bc2 = bitcast i32 %and to float + ret float %bc2 +} + +; Make sure that doubles work too. + +define double @doubles(double %x, double %y) { +; CHECK-LABEL: doubles: +; CHECK: # BB#0: +; CHECK-NEXT: andpd %xmm1, %xmm0 +; CHECK-NEXT: retq + + %bc1 = bitcast double %x to i64 + %bc2 = bitcast double %y to i64 + %and = and i64 %bc1, %bc2 + %bc3 = bitcast i64 %and to double + ret double %bc3 +} + +define double @f7_double(double %x) { +; CHECK-LABEL: f7_double: +; CHECK: # BB#0: +; CHECK-NEXT: movsd {{.*#+}} xmm1 = mem[0],zero +; CHECK-NEXT: andpd %xmm1, %xmm0 +; CHECK-NEXT: retq + + %bc1 = bitcast double %x to i64 + %and = and i64 %bc1, 3 + %bc2 = bitcast i64 %and to double + ret double %bc2 +} + +; Grabbing the sign bit is a special case that could be handled +; by movmskps/movmskpd, but if we're not shifting it over, then +; a simple FP logic op is cheaper. + +define float @movmsk(float %x) { +; CHECK-LABEL: movmsk: +; CHECK: # BB#0: +; CHECK-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero +; CHECK-NEXT: andps %xmm1, %xmm0 +; CHECK-NEXT: retq + + %bc1 = bitcast float %x to i32 + %and = and i32 %bc1, 2147483648 + %bc2 = bitcast i32 %and to float + ret float %bc2 +} + diff --git a/test/CodeGen/X86/fp128-calling-conv.ll b/test/CodeGen/X86/fp128-calling-conv.ll new file mode 100644 index 000000000000..e1dab30847c8 --- /dev/null +++ b/test/CodeGen/X86/fp128-calling-conv.ll @@ -0,0 +1,47 @@ +; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+mmx | FileCheck %s +; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+mmx | FileCheck %s + +; __float128 myFP128 = 1.0L; // x86_64-linux-android +@myFP128 = global fp128 0xL00000000000000003FFF000000000000, align 16 + +; The first few parameters are passed in registers and the other are on stack. + +define fp128 @TestParam_FP128_0(fp128 %d0, fp128 %d1, fp128 %d2, fp128 %d3, fp128 %d4, fp128 %d5, fp128 %d6, fp128 %d7, fp128 %d8, fp128 %d9, fp128 %d10, fp128 %d11, fp128 %d12, fp128 %d13, fp128 %d14, fp128 %d15, fp128 %d16, fp128 %d17, fp128 %d18, fp128 %d19) { +entry: + ret fp128 %d0 +; CHECK-LABEL: TestParam_FP128_0: +; CHECK-NOT: mov +; CHECK: retq +} + +define fp128 @TestParam_FP128_1(fp128 %d0, fp128 %d1, fp128 %d2, fp128 %d3, fp128 %d4, fp128 %d5, fp128 %d6, fp128 %d7, fp128 %d8, fp128 %d9, fp128 %d10, fp128 %d11, fp128 %d12, fp128 %d13, fp128 %d14, fp128 %d15, fp128 %d16, fp128 %d17, fp128 %d18, fp128 %d19) { +entry: + ret fp128 %d1 +; CHECK-LABEL: TestParam_FP128_1: +; CHECK: movaps %xmm1, %xmm0 +; CHECK-NEXT: retq +} + +define fp128 @TestParam_FP128_7(fp128 %d0, fp128 %d1, fp128 %d2, fp128 %d3, fp128 %d4, fp128 %d5, fp128 %d6, fp128 %d7, fp128 %d8, fp128 %d9, fp128 %d10, fp128 %d11, fp128 %d12, fp128 %d13, fp128 %d14, fp128 %d15, fp128 %d16, fp128 %d17, fp128 %d18, fp128 %d19) { +entry: + ret fp128 %d7 +; CHECK-LABEL: TestParam_FP128_7: +; CHECK: movaps %xmm7, %xmm0 +; CHECK-NEXT: retq +} + +define fp128 @TestParam_FP128_8(fp128 %d0, fp128 %d1, fp128 %d2, fp128 %d3, fp128 %d4, fp128 %d5, fp128 %d6, fp128 %d7, fp128 %d8, fp128 %d9, fp128 %d10, fp128 %d11, fp128 %d12, fp128 %d13, fp128 %d14, fp128 %d15, fp128 %d16, fp128 %d17, fp128 %d18, fp128 %d19) { +entry: + ret fp128 %d8 +; CHECK-LABEL: TestParam_FP128_8: +; CHECK: movaps 8(%rsp), %xmm0 +; CHECK-NEXT: retq +} + +define fp128 @TestParam_FP128_9(fp128 %d0, fp128 %d1, fp128 %d2, fp128 %d3, fp128 %d4, fp128 %d5, fp128 %d6, fp128 %d7, fp128 %d8, fp128 %d9, fp128 %d10, fp128 %d11, fp128 %d12, fp128 %d13, fp128 %d14, fp128 %d15, fp128 %d16, fp128 %d17, fp128 %d18, fp128 %d19) { +entry: + ret fp128 %d9 +; CHECK-LABEL: TestParam_FP128_9: +; CHECK: movaps 24(%rsp), %xmm0 +; CHECK-NEXT: retq +} diff --git a/test/CodeGen/X86/fp128-cast.ll b/test/CodeGen/X86/fp128-cast.ll new file mode 100644 index 000000000000..73878e31d0ef --- /dev/null +++ b/test/CodeGen/X86/fp128-cast.ll @@ -0,0 +1,279 @@ +; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+mmx | FileCheck %s +; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+mmx | FileCheck %s + +; Check soft floating point conversion function calls. + +@vi32 = common global i32 0, align 4 +@vi64 = common global i64 0, align 8 +@vu32 = common global i32 0, align 4 +@vu64 = common global i64 0, align 8 +@vf32 = common global float 0.000000e+00, align 4 +@vf64 = common global double 0.000000e+00, align 8 +@vf128 = common global fp128 0xL00000000000000000000000000000000, align 16 + +define void @TestFPExtF32_F128() { +entry: + %0 = load float, float* @vf32, align 4 + %conv = fpext float %0 to fp128 + store fp128 %conv, fp128* @vf128, align 16 + ret void +; CHECK-LABEL: TestFPExtF32_F128: +; CHECK: movss vf32(%rip), %xmm0 +; CHECK-NEXT: callq __extendsftf2 +; CHECK-NEXT: movaps %xmm0, vf128(%rip) +; CHECK: retq +} + +define void @TestFPExtF64_F128() { +entry: + %0 = load double, double* @vf64, align 8 + %conv = fpext double %0 to fp128 + store fp128 %conv, fp128* @vf128, align 16 + ret void +; CHECK-LABEL: TestFPExtF64_F128: +; CHECK: movsd vf64(%rip), %xmm0 +; CHECK-NEXT: callq __extenddftf2 +; CHECK-NEXT: movapd %xmm0, vf128(%rip) +; CHECK: ret +} + +define void @TestFPToSIF128_I32() { +entry: + %0 = load fp128, fp128* @vf128, align 16 + %conv = fptosi fp128 %0 to i32 + store i32 %conv, i32* @vi32, align 4 + ret void +; CHECK-LABEL: TestFPToSIF128_I32: +; CHECK: movaps vf128(%rip), %xmm0 +; CHECK-NEXT: callq __fixtfsi +; CHECK-NEXT: movl %eax, vi32(%rip) +; CHECK: retq +} + +define void @TestFPToUIF128_U32() { +entry: + %0 = load fp128, fp128* @vf128, align 16 + %conv = fptoui fp128 %0 to i32 + store i32 %conv, i32* @vu32, align 4 + ret void +; CHECK-LABEL: TestFPToUIF128_U32: +; CHECK: movaps vf128(%rip), %xmm0 +; CHECK-NEXT: callq __fixunstfsi +; CHECK-NEXT: movl %eax, vu32(%rip) +; CHECK: retq +} + +define void @TestFPToSIF128_I64() { +entry: + %0 = load fp128, fp128* @vf128, align 16 + %conv = fptosi fp128 %0 to i32 + %conv1 = sext i32 %conv to i64 + store i64 %conv1, i64* @vi64, align 8 + ret void +; CHECK-LABEL: TestFPToSIF128_I64: +; CHECK: movaps vf128(%rip), %xmm0 +; CHECK-NEXT: callq __fixtfsi +; CHECK-NEXT: cltq +; CHECK-NEXT: movq %rax, vi64(%rip) +; CHECK: retq +} + +define void @TestFPToUIF128_U64() { +entry: + %0 = load fp128, fp128* @vf128, align 16 + %conv = fptoui fp128 %0 to i32 + %conv1 = zext i32 %conv to i64 + store i64 %conv1, i64* @vu64, align 8 + ret void +; CHECK-LABEL: TestFPToUIF128_U64: +; CHECK: movaps vf128(%rip), %xmm0 +; CHECK-NEXT: callq __fixunstfsi +; CHECK-NEXT: movl %eax, %eax +; CHECK-NEXT: movq %rax, vu64(%rip) +; CHECK: retq +} + +define void @TestFPTruncF128_F32() { +entry: + %0 = load fp128, fp128* @vf128, align 16 + %conv = fptrunc fp128 %0 to float + store float %conv, float* @vf32, align 4 + ret void +; CHECK-LABEL: TestFPTruncF128_F32: +; CHECK: movaps vf128(%rip), %xmm0 +; CHECK-NEXT: callq __trunctfsf2 +; CHECK-NEXT: movss %xmm0, vf32(%rip) +; CHECK: retq +} + +define void @TestFPTruncF128_F64() { +entry: + %0 = load fp128, fp128* @vf128, align 16 + %conv = fptrunc fp128 %0 to double + store double %conv, double* @vf64, align 8 + ret void +; CHECK-LABEL: TestFPTruncF128_F64: +; CHECK: movapd vf128(%rip), %xmm0 +; CHECK-NEXT: callq __trunctfdf2 +; CHECK-NEXT: movsd %xmm0, vf64(%rip) +; CHECK: retq +} + +define void @TestSIToFPI32_F128() { +entry: + %0 = load i32, i32* @vi32, align 4 + %conv = sitofp i32 %0 to fp128 + store fp128 %conv, fp128* @vf128, align 16 + ret void +; CHECK-LABEL: TestSIToFPI32_F128: +; CHECK: movl vi32(%rip), %edi +; CHECK-NEXT: callq __floatsitf +; CHECK-NEXT: movaps %xmm0, vf128(%rip) +; CHECK: retq +} + +define void @TestUIToFPU32_F128() #2 { +entry: + %0 = load i32, i32* @vu32, align 4 + %conv = uitofp i32 %0 to fp128 + store fp128 %conv, fp128* @vf128, align 16 + ret void +; CHECK-LABEL: TestUIToFPU32_F128: +; CHECK: movl vu32(%rip), %edi +; CHECK-NEXT: callq __floatunsitf +; CHECK-NEXT: movaps %xmm0, vf128(%rip) +; CHECK: retq +} + +define void @TestSIToFPI64_F128(){ +entry: + %0 = load i64, i64* @vi64, align 8 + %conv = sitofp i64 %0 to fp128 + store fp128 %conv, fp128* @vf128, align 16 + ret void +; CHECK-LABEL: TestSIToFPI64_F128: +; CHECK: movq vi64(%rip), %rdi +; CHECK-NEXT: callq __floatditf +; CHECK-NEXT: movaps %xmm0, vf128(%rip) +; CHECK: retq +} + +define void @TestUIToFPU64_F128() #2 { +entry: + %0 = load i64, i64* @vu64, align 8 + %conv = uitofp i64 %0 to fp128 + store fp128 %conv, fp128* @vf128, align 16 + ret void +; CHECK-LABEL: TestUIToFPU64_F128: +; CHECK: movq vu64(%rip), %rdi +; CHECK-NEXT: callq __floatunditf +; CHECK-NEXT: movaps %xmm0, vf128(%rip) +; CHECK: retq +} + +define i32 @TestConst128(fp128 %v) { +entry: + %cmp = fcmp ogt fp128 %v, 0xL00000000000000003FFF000000000000 + %conv = zext i1 %cmp to i32 + ret i32 %conv +; CHECK-LABEL: TestConst128: +; CHECK: movaps {{.*}}, %xmm1 +; CHECK-NEXT: callq __gttf2 +; CHECK-NEXT: test +; CHECK: retq +} + +; C code: +; struct TestBits_ieee_ext { +; unsigned v1; +; unsigned v2; +; }; +; union TestBits_LDU { +; FP128 ld; +; struct TestBits_ieee_ext bits; +; }; +; int TestBits128(FP128 ld) { +; union TestBits_LDU u; +; u.ld = ld * ld; +; return ((u.bits.v1 | u.bits.v2) == 0); +; } +define i32 @TestBits128(fp128 %ld) { +entry: + %mul = fmul fp128 %ld, %ld + %0 = bitcast fp128 %mul to i128 + %shift = lshr i128 %0, 32 + %or5 = or i128 %shift, %0 + %or = trunc i128 %or5 to i32 + %cmp = icmp eq i32 %or, 0 + %conv = zext i1 %cmp to i32 + ret i32 %conv +; CHECK-LABEL: TestBits128: +; CHECK: movaps %xmm0, %xmm1 +; CHECK-NEXT: callq __multf3 +; CHECK-NEXT: movaps %xmm0, (%rsp) +; CHECK-NEXT: movq (%rsp), +; CHECK-NEXT: movq % +; CHECK-NEXT: shrq $32, +; CHECK: orl +; CHECK-NEXT: sete %al +; CHECK-NEXT: movzbl %al, %eax +; CHECK: retq +; +; If TestBits128 fails due to any llvm or clang change, +; please make sure the original simplified C code will +; be compiled into correct IL and assembly code, not +; just this TestBits128 test case. Better yet, try to +; test the whole libm and its test cases. +} + +; C code: (compiled with -target x86_64-linux-android) +; typedef long double __float128; +; __float128 TestPair128(unsigned long a, unsigned long b) { +; unsigned __int128 n; +; unsigned __int128 v1 = ((unsigned __int128)a << 64); +; unsigned __int128 v2 = (unsigned __int128)b; +; n = (v1 | v2) + 3; +; return *(__float128*)&n; +; } +define fp128 @TestPair128(i64 %a, i64 %b) { +entry: + %conv = zext i64 %a to i128 + %shl = shl nuw i128 %conv, 64 + %conv1 = zext i64 %b to i128 + %or = or i128 %shl, %conv1 + %add = add i128 %or, 3 + %0 = bitcast i128 %add to fp128 + ret fp128 %0 +; CHECK-LABEL: TestPair128: +; CHECK: addq $3, %rsi +; CHECK: movq %rsi, -24(%rsp) +; CHECK: movq %rdi, -16(%rsp) +; CHECK: movaps -24(%rsp), %xmm0 +; CHECK-NEXT: retq +} + +define fp128 @TestTruncCopysign(fp128 %x, i32 %n) { +entry: + %cmp = icmp sgt i32 %n, 50000 + br i1 %cmp, label %if.then, label %cleanup + +if.then: ; preds = %entry + %conv = fptrunc fp128 %x to double + %call = tail call double @copysign(double 0x7FF0000000000000, double %conv) #2 + %conv1 = fpext double %call to fp128 + br label %cleanup + +cleanup: ; preds = %entry, %if.then + %retval.0 = phi fp128 [ %conv1, %if.then ], [ %x, %entry ] + ret fp128 %retval.0 +; CHECK-LABEL: TestTruncCopysign: +; CHECK: callq __trunctfdf2 +; CHECK-NEXT: andpd {{.*}}, %xmm0 +; CHECK-NEXT: orpd {{.*}}, %xmm0 +; CHECK-NEXT: callq __extenddftf2 +; CHECK: retq +} + +declare double @copysign(double, double) #1 + +attributes #2 = { nounwind readnone } diff --git a/test/CodeGen/X86/fp128-compare.ll b/test/CodeGen/X86/fp128-compare.ll new file mode 100644 index 000000000000..b5d4fbe1b74e --- /dev/null +++ b/test/CodeGen/X86/fp128-compare.ll @@ -0,0 +1,96 @@ +; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+mmx | FileCheck %s +; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+mmx | FileCheck %s + +define i32 @TestComp128GT(fp128 %d1, fp128 %d2) { +entry: + %cmp = fcmp ogt fp128 %d1, %d2 + %conv = zext i1 %cmp to i32 + ret i32 %conv +; CHECK-LABEL: TestComp128GT: +; CHECK: callq __gttf2 +; CHECK: setg %al +; CHECK: movzbl %al, %eax +; CHECK: retq +} + +define i32 @TestComp128GE(fp128 %d1, fp128 %d2) { +entry: + %cmp = fcmp oge fp128 %d1, %d2 + %conv = zext i1 %cmp to i32 + ret i32 %conv +; CHECK-LABEL: TestComp128GE: +; CHECK: callq __getf2 +; CHECK: testl %eax, %eax +; CHECK: setns %al +; CHECK: movzbl %al, %eax +; CHECK: retq +} + +define i32 @TestComp128LT(fp128 %d1, fp128 %d2) { +entry: + %cmp = fcmp olt fp128 %d1, %d2 + %conv = zext i1 %cmp to i32 + ret i32 %conv +; CHECK-LABEL: TestComp128LT: +; CHECK: callq __lttf2 +; CHECK-NEXT: shrl $31, %eax +; CHECK: retq +; +; The 'shrl' is a special optimization in llvm to combine +; the effect of 'fcmp olt' and 'zext'. The main purpose is +; to test soften call to __lttf2. +} + +define i32 @TestComp128LE(fp128 %d1, fp128 %d2) { +entry: + %cmp = fcmp ole fp128 %d1, %d2 + %conv = zext i1 %cmp to i32 + ret i32 %conv +; CHECK-LABEL: TestComp128LE: +; CHECK: callq __letf2 +; CHECK-NEXT: testl %eax, %eax +; CHECK: setle %al +; CHECK: movzbl %al, %eax +; CHECK: retq +} + +define i32 @TestComp128EQ(fp128 %d1, fp128 %d2) { +entry: + %cmp = fcmp oeq fp128 %d1, %d2 + %conv = zext i1 %cmp to i32 + ret i32 %conv +; CHECK-LABEL: TestComp128EQ: +; CHECK: callq __eqtf2 +; CHECK-NEXT: testl %eax, %eax +; CHECK: sete %al +; CHECK: movzbl %al, %eax +; CHECK: retq +} + +define i32 @TestComp128NE(fp128 %d1, fp128 %d2) { +entry: + %cmp = fcmp une fp128 %d1, %d2 + %conv = zext i1 %cmp to i32 + ret i32 %conv +; CHECK-LABEL: TestComp128NE: +; CHECK: callq __netf2 +; CHECK-NEXT: testl %eax, %eax +; CHECK: setne %al +; CHECK: movzbl %al, %eax +; CHECK: retq +} + +define fp128 @TestMax(fp128 %x, fp128 %y) { +entry: + %cmp = fcmp ogt fp128 %x, %y + %cond = select i1 %cmp, fp128 %x, fp128 %y + ret fp128 %cond +; CHECK-LABEL: TestMax: +; CHECK: movaps %xmm1 +; CHECK: movaps %xmm0 +; CHECK: callq __gttf2 +; CHECK: movaps {{.*}}, %xmm0 +; CHECK: testl %eax, %eax +; CHECK: movaps {{.*}}, %xmm0 +; CHECK: retq +} diff --git a/test/CodeGen/X86/fp128-i128.ll b/test/CodeGen/X86/fp128-i128.ll new file mode 100644 index 000000000000..77160674ab20 --- /dev/null +++ b/test/CodeGen/X86/fp128-i128.ll @@ -0,0 +1,320 @@ +; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+mmx | FileCheck %s +; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+mmx | FileCheck %s + +; These tests were generated from simplified libm C code. +; When compiled for the x86_64-linux-android target, +; long double is mapped to f128 type that should be passed +; in SSE registers. When the f128 type calling convention +; problem was fixed, old llvm code failed to handle f128 values +; in several f128/i128 type operations. These unit tests hopefully +; will catch regression in any future change in this area. +; To modified or enhance these test cases, please consult libm +; code pattern and compile with -target x86_64-linux-android +; to generate IL. The __float128 keyword if not accepted by +; clang, just define it to "long double". +; + +; typedef long double __float128; +; union IEEEl2bits { +; __float128 e; +; struct { +; unsigned long manl :64; +; unsigned long manh :48; +; unsigned int exp :15; +; unsigned int sign :1; +; } bits; +; struct { +; unsigned long manl :64; +; unsigned long manh :48; +; unsigned int expsign :16; +; } xbits; +; }; + +; C code: +; void foo(__float128 x); +; void TestUnionLD1(__float128 s, unsigned long n) { +; union IEEEl2bits u; +; __float128 w; +; u.e = s; +; u.bits.manh = n; +; w = u.e; +; foo(w); +; } +define void @TestUnionLD1(fp128 %s, i64 %n) #0 { +entry: + %0 = bitcast fp128 %s to i128 + %1 = zext i64 %n to i128 + %bf.value = shl nuw i128 %1, 64 + %bf.shl = and i128 %bf.value, 5192296858534809181786422619668480 + %bf.clear = and i128 %0, -5192296858534809181786422619668481 + %bf.set = or i128 %bf.shl, %bf.clear + %2 = bitcast i128 %bf.set to fp128 + tail call void @foo(fp128 %2) #2 + ret void +; CHECK-LABEL: TestUnionLD1: +; CHECK: movaps %xmm0, -24(%rsp) +; CHECK-NEXT: movq -24(%rsp), %rax +; CHECK-NEXT: movabsq $281474976710655, %rcx +; CHECK-NEXT: andq %rdi, %rcx +; CHECK-NEXT: movabsq $-281474976710656, %rdx +; CHECK-NEXT: andq -16(%rsp), %rdx +; CHECK-NEXT: movq %rax, -40(%rsp) +; CHECK-NEXT: orq %rcx, %rdx +; CHECK-NEXT: movq %rdx, -32(%rsp) +; CHECK-NEXT: movaps -40(%rsp), %xmm0 +; CHECK-NEXT: jmp foo +} + +; C code: +; __float128 TestUnionLD2(__float128 s) { +; union IEEEl2bits u; +; __float128 w; +; u.e = s; +; u.bits.manl = 0; +; w = u.e; +; return w; +; } +define fp128 @TestUnionLD2(fp128 %s) #0 { +entry: + %0 = bitcast fp128 %s to i128 + %bf.clear = and i128 %0, -18446744073709551616 + %1 = bitcast i128 %bf.clear to fp128 + ret fp128 %1 +; CHECK-LABEL: TestUnionLD2: +; CHECK: movaps %xmm0, -24(%rsp) +; CHECK-NEXT: movq -16(%rsp), %rax +; CHECK-NEXT: movq %rax, -32(%rsp) +; CHECK-NEXT: movq $0, -40(%rsp) +; CHECK-NEXT: movaps -40(%rsp), %xmm0 +; CHECK-NEXT: retq +} + +; C code: +; __float128 TestI128_1(__float128 x) +; { +; union IEEEl2bits z; +; z.e = x; +; z.bits.sign = 0; +; return (z.e < 0.1L) ? 1.0L : 2.0L; +; } +define fp128 @TestI128_1(fp128 %x) #0 { +entry: + %0 = bitcast fp128 %x to i128 + %bf.clear = and i128 %0, 170141183460469231731687303715884105727 + %1 = bitcast i128 %bf.clear to fp128 + %cmp = fcmp olt fp128 %1, 0xL999999999999999A3FFB999999999999 + %cond = select i1 %cmp, fp128 0xL00000000000000003FFF000000000000, fp128 0xL00000000000000004000000000000000 + ret fp128 %cond +; CHECK-LABEL: TestI128_1: +; CHECK: movaps %xmm0, +; CHECK: movabsq $9223372036854775807, +; CHECK: callq __lttf2 +; CHECK: testl %eax, %eax +; CHECK: movaps {{.*}}, %xmm0 +; CHECK: retq +} + +; C code: +; __float128 TestI128_2(__float128 x, __float128 y) +; { +; unsigned short hx; +; union IEEEl2bits ge_u; +; ge_u.e = x; +; hx = ge_u.xbits.expsign; +; return (hx & 0x8000) == 0 ? x : y; +; } +define fp128 @TestI128_2(fp128 %x, fp128 %y) #0 { +entry: + %0 = bitcast fp128 %x to i128 + %cmp = icmp sgt i128 %0, -1 + %cond = select i1 %cmp, fp128 %x, fp128 %y + ret fp128 %cond +; CHECK-LABEL: TestI128_2: +; CHECK: movaps %xmm0, -24(%rsp) +; CHECK-NEXT: cmpq $0, -16(%rsp) +; CHECK-NEXT: jns +; CHECK: movaps %xmm1, %xmm0 +; CHECK: retq +} + +; C code: +; __float128 TestI128_3(__float128 x, int *ex) +; { +; union IEEEl2bits u; +; u.e = x; +; if (u.bits.exp == 0) { +; u.e *= 0x1.0p514; +; u.bits.exp = 0x3ffe; +; } +; return (u.e); +; } +define fp128 @TestI128_3(fp128 %x, i32* nocapture readnone %ex) #0 { +entry: + %0 = bitcast fp128 %x to i128 + %bf.cast = and i128 %0, 170135991163610696904058773219554885632 + %cmp = icmp eq i128 %bf.cast, 0 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + %mul = fmul fp128 %x, 0xL00000000000000004201000000000000 + %1 = bitcast fp128 %mul to i128 + %bf.clear4 = and i128 %1, -170135991163610696904058773219554885633 + %bf.set = or i128 %bf.clear4, 85060207136517546210586590865283612672 + br label %if.end + +if.end: ; preds = %if.then, %entry + %u.sroa.0.0 = phi i128 [ %bf.set, %if.then ], [ %0, %entry ] + %2 = bitcast i128 %u.sroa.0.0 to fp128 + ret fp128 %2 +; CHECK-LABEL: TestI128_3: +; CHECK: movaps %xmm0, +; CHECK: movabsq $9223090561878065152, +; CHECK: testq +; CHECK: callq __multf3 +; CHECK-NEXT: movaps %xmm0 +; CHECK: movabsq $-9223090561878065153, +; CHECK: movabsq $4611123068473966592, +; CHECK: retq +} + +; C code: +; __float128 TestI128_4(__float128 x) +; { +; union IEEEl2bits u; +; __float128 df; +; u.e = x; +; u.xbits.manl = 0; +; df = u.e; +; return x + df; +; } +define fp128 @TestI128_4(fp128 %x) #0 { +entry: + %0 = bitcast fp128 %x to i128 + %bf.clear = and i128 %0, -18446744073709551616 + %1 = bitcast i128 %bf.clear to fp128 + %add = fadd fp128 %1, %x + ret fp128 %add +; CHECK-LABEL: TestI128_4: +; CHECK: movaps %xmm0, %xmm1 +; CHECK-NEXT: movaps %xmm1, 16(%rsp) +; CHECK-NEXT: movq 24(%rsp), %rax +; CHECK-NEXT: movq %rax, 8(%rsp) +; CHECK-NEXT: movq $0, (%rsp) +; CHECK-NEXT: movaps (%rsp), %xmm0 +; CHECK-NEXT: callq __addtf3 +; CHECK: retq +} + +@v128 = common global i128 0, align 16 +@v128_2 = common global i128 0, align 16 + +; C code: +; unsigned __int128 v128, v128_2; +; void TestShift128_2() { +; v128 = ((v128 << 96) | v128_2); +; } +define void @TestShift128_2() #2 { +entry: + %0 = load i128, i128* @v128, align 16 + %shl = shl i128 %0, 96 + %1 = load i128, i128* @v128_2, align 16 + %or = or i128 %shl, %1 + store i128 %or, i128* @v128, align 16 + ret void +; CHECK-LABEL: TestShift128_2: +; CHECK: movq v128(%rip), %rax +; CHECK-NEXT: shlq $32, %rax +; CHECK-NEXT: movq v128_2(%rip), %rcx +; CHECK-NEXT: orq v128_2+8(%rip), %rax +; CHECK-NEXT: movq %rcx, v128(%rip) +; CHECK-NEXT: movq %rax, v128+8(%rip) +; CHECK-NEXT: retq +} + +define fp128 @acosl(fp128 %x) #0 { +entry: + %0 = bitcast fp128 %x to i128 + %bf.clear = and i128 %0, -18446744073709551616 + %1 = bitcast i128 %bf.clear to fp128 + %add = fadd fp128 %1, %x + ret fp128 %add +; CHECK-LABEL: acosl: +; CHECK: movaps %xmm0, %xmm1 +; CHECK-NEXT: movaps %xmm1, 16(%rsp) +; CHECK-NEXT: movq 24(%rsp), %rax +; CHECK-NEXT: movq %rax, 8(%rsp) +; CHECK-NEXT: movq $0, (%rsp) +; CHECK-NEXT: movaps (%rsp), %xmm0 +; CHECK-NEXT: callq __addtf3 +; CHECK: retq +} + +; Compare i128 values and check i128 constants. +define fp128 @TestComp(fp128 %x, fp128 %y) #0 { +entry: + %0 = bitcast fp128 %x to i128 + %cmp = icmp sgt i128 %0, -1 + %cond = select i1 %cmp, fp128 %x, fp128 %y + ret fp128 %cond +; CHECK-LABEL: TestComp: +; CHECK: movaps %xmm0, -24(%rsp) +; CHECK-NEXT: cmpq $0, -16(%rsp) +; CHECK-NEXT: jns +; CHECK: movaps %xmm1, %xmm0 +; CHECK: retq +} + +declare void @foo(fp128) #1 + +; Test logical operations on fp128 values. +define fp128 @TestFABS_LD(fp128 %x) #0 { +entry: + %call = tail call fp128 @fabsl(fp128 %x) #2 + ret fp128 %call +; CHECK-LABEL: TestFABS_LD +; CHECK: andps {{.*}}, %xmm0 +; CHECK-NEXT: retq +} + +declare fp128 @fabsl(fp128) #1 + +declare fp128 @copysignl(fp128, fp128) #1 + +; Test more complicated logical operations generated from copysignl. +define void @TestCopySign({ fp128, fp128 }* noalias nocapture sret %agg.result, { fp128, fp128 }* byval nocapture readonly align 16 %z) #0 { +entry: + %z.realp = getelementptr inbounds { fp128, fp128 }, { fp128, fp128 }* %z, i64 0, i32 0 + %z.real = load fp128, fp128* %z.realp, align 16 + %z.imagp = getelementptr inbounds { fp128, fp128 }, { fp128, fp128 }* %z, i64 0, i32 1 + %z.imag4 = load fp128, fp128* %z.imagp, align 16 + %cmp = fcmp ogt fp128 %z.real, %z.imag4 + %sub = fsub fp128 %z.imag4, %z.imag4 + br i1 %cmp, label %if.then, label %cleanup + +if.then: ; preds = %entry + %call = tail call fp128 @fabsl(fp128 %sub) #2 + br label %cleanup + +cleanup: ; preds = %entry, %if.then + %z.real.sink = phi fp128 [ %z.real, %if.then ], [ %sub, %entry ] + %call.sink = phi fp128 [ %call, %if.then ], [ %z.real, %entry ] + %call5 = tail call fp128 @copysignl(fp128 %z.real.sink, fp128 %z.imag4) #2 + %0 = getelementptr inbounds { fp128, fp128 }, { fp128, fp128 }* %agg.result, i64 0, i32 0 + %1 = getelementptr inbounds { fp128, fp128 }, { fp128, fp128 }* %agg.result, i64 0, i32 1 + store fp128 %call.sink, fp128* %0, align 16 + store fp128 %call5, fp128* %1, align 16 + ret void +; CHECK-LABEL: TestCopySign +; CHECK-NOT: call +; CHECK: callq __subtf3 +; CHECK-NOT: call +; CHECK: callq __gttf2 +; CHECK-NOT: call +; CHECK: andps {{.*}}, %xmm0 +; CHECK: retq +} + + +attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+ssse3,+sse3,+popcnt,+sse,+sse2,+sse4.1,+sse4.2" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+ssse3,+sse3,+popcnt,+sse,+sse2,+sse4.1,+sse4.2" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #2 = { nounwind readnone } diff --git a/test/CodeGen/X86/fp128-libcalls.ll b/test/CodeGen/X86/fp128-libcalls.ll new file mode 100644 index 000000000000..ee5fa447448c --- /dev/null +++ b/test/CodeGen/X86/fp128-libcalls.ll @@ -0,0 +1,107 @@ +; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+mmx | FileCheck %s +; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+mmx | FileCheck %s + +; Check all soft floating point library function calls. + +@vf64 = common global double 0.000000e+00, align 8 +@vf128 = common global fp128 0xL00000000000000000000000000000000, align 16 + +define void @Test128Add(fp128 %d1, fp128 %d2) { +entry: + %add = fadd fp128 %d1, %d2 + store fp128 %add, fp128* @vf128, align 16 + ret void +; CHECK-LABEL: Test128Add: +; CHECK: callq __addtf3 +; CHECK-NEXT: movaps %xmm0, vf128(%rip) +; CHECK: retq +} + +define void @Test128_1Add(fp128 %d1){ +entry: + %0 = load fp128, fp128* @vf128, align 16 + %add = fadd fp128 %0, %d1 + store fp128 %add, fp128* @vf128, align 16 + ret void +; CHECK-LABEL: Test128_1Add: +; CHECK: movaps %xmm0, %xmm1 +; CHECK-NEXT: movaps vf128(%rip), %xmm0 +; CHECK-NEXT: callq __addtf3 +; CHECK-NEXT: movaps %xmm0, vf128(%rip) +; CHECK: retq +} + +define void @Test128Sub(fp128 %d1, fp128 %d2){ +entry: + %sub = fsub fp128 %d1, %d2 + store fp128 %sub, fp128* @vf128, align 16 + ret void +; CHECK-LABEL: Test128Sub: +; CHECK: callq __subtf3 +; CHECK-NEXT: movaps %xmm0, vf128(%rip) +; CHECK: retq +} + +define void @Test128_1Sub(fp128 %d1){ +entry: + %0 = load fp128, fp128* @vf128, align 16 + %sub = fsub fp128 %0, %d1 + store fp128 %sub, fp128* @vf128, align 16 + ret void +; CHECK-LABEL: Test128_1Sub: +; CHECK: movaps %xmm0, %xmm1 +; CHECK-NEXT: movaps vf128(%rip), %xmm0 +; CHECK-NEXT: callq __subtf3 +; CHECK-NEXT: movaps %xmm0, vf128(%rip) +; CHECK: retq +} + +define void @Test128Mul(fp128 %d1, fp128 %d2){ +entry: + %mul = fmul fp128 %d1, %d2 + store fp128 %mul, fp128* @vf128, align 16 + ret void +; CHECK-LABEL: Test128Mul: +; CHECK: callq __multf3 +; CHECK-NEXT: movaps %xmm0, vf128(%rip) +; CHECK: retq +} + +define void @Test128_1Mul(fp128 %d1){ +entry: + %0 = load fp128, fp128* @vf128, align 16 + %mul = fmul fp128 %0, %d1 + store fp128 %mul, fp128* @vf128, align 16 + ret void +; CHECK-LABEL: Test128_1Mul: +; CHECK: movaps %xmm0, %xmm1 +; CHECK-NEXT: movaps vf128(%rip), %xmm0 +; CHECK-NEXT: callq __multf3 +; CHECK-NEXT: movaps %xmm0, vf128(%rip) +; CHECK: retq +} + +define void @Test128Div(fp128 %d1, fp128 %d2){ +entry: + %div = fdiv fp128 %d1, %d2 + store fp128 %div, fp128* @vf128, align 16 + ret void +; CHECK-LABEL: Test128Div: +; CHECK: callq __divtf3 +; CHECK-NEXT: movaps %xmm0, vf128(%rip) +; CHECK: retq +} + +define void @Test128_1Div(fp128 %d1){ +entry: + %0 = load fp128, fp128* @vf128, align 16 + %div = fdiv fp128 %0, %d1 + store fp128 %div, fp128* @vf128, align 16 + ret void +; CHECK-LABEL: Test128_1Div: +; CHECK: movaps %xmm0, %xmm1 +; CHECK-NEXT: movaps vf128(%rip), %xmm0 +; CHECK-NEXT: callq __divtf3 +; CHECK-NEXT: movaps %xmm0, vf128(%rip) +; CHECK: retq +} diff --git a/test/CodeGen/X86/fp128-load.ll b/test/CodeGen/X86/fp128-load.ll new file mode 100644 index 000000000000..73bacf87275e --- /dev/null +++ b/test/CodeGen/X86/fp128-load.ll @@ -0,0 +1,35 @@ +; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+mmx | FileCheck %s +; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+mmx | FileCheck %s + +; __float128 myFP128 = 1.0L; // x86_64-linux-android +@my_fp128 = global fp128 0xL00000000000000003FFF000000000000, align 16 + +define fp128 @get_fp128() { +entry: + %0 = load fp128, fp128* @my_fp128, align 16 + ret fp128 %0 +; CHECK-LABEL: get_fp128: +; CHECK: movaps my_fp128(%rip), %xmm0 +; CHECK-NEXT: retq +} + +@TestLoadExtend.data = internal unnamed_addr constant [2 x float] [float 0x3FB99999A0000000, float 0x3FC99999A0000000], align 4 + +define fp128 @TestLoadExtend(fp128 %x, i32 %n) { +entry: + %idxprom = sext i32 %n to i64 + %arrayidx = getelementptr inbounds [2 x float], [2 x float]* @TestLoadExtend.data, i64 0, i64 %idxprom + %0 = load float, float* %arrayidx, align 4 + %conv = fpext float %0 to fp128 + ret fp128 %conv +; CHECK-LABEL: TestLoadExtend: +; CHECK: movslq %edi, %rax +; CHECK-NEXT: movss TestLoadExtend.data(,%rax,4), %xmm0 +; CHECK-NEXT: callq __extendsftf2 +; CHECK: retq +} + +; CHECK-LABEL: my_fp128: +; CHECK-NEXT: .quad 0 +; CHECK-NEXT: .quad 4611404543450677248 +; CHECK-NEXT: .size my_fp128, 16 diff --git a/test/CodeGen/X86/fp128-store.ll b/test/CodeGen/X86/fp128-store.ll new file mode 100644 index 000000000000..ca3af637cff5 --- /dev/null +++ b/test/CodeGen/X86/fp128-store.ll @@ -0,0 +1,14 @@ +; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+mmx | FileCheck %s +; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+mmx | FileCheck %s + +; __float128 myFP128 = 1.0L; // x86_64-linux-android +@myFP128 = global fp128 0xL00000000000000003FFF000000000000, align 16 + +define void @set_FP128(fp128 %x) { +entry: + store fp128 %x, fp128* @myFP128, align 16 + ret void +; CHECK-LABEL: set_FP128: +; CHECK: movaps %xmm0, myFP128(%rip) +; CHECK-NEXT: retq +} diff --git a/test/CodeGen/X86/fpcmp-soft-fp.ll b/test/CodeGen/X86/fpcmp-soft-fp.ll new file mode 100644 index 000000000000..58d57017d18a --- /dev/null +++ b/test/CodeGen/X86/fpcmp-soft-fp.ll @@ -0,0 +1,127 @@ +; RUN: llc < %s -march=x86 -mcpu=pentium -mtriple=x86-linux-gnu -float-abi=soft | FileCheck %s + +define i1 @test1(double %d) #0 { +entry: + %cmp = fcmp ule double %d, 0.000000e+00 + ret i1 %cmp +} +; CHECK-LABEL: test1: +; CHECK: calll __gtdf2 +; CHECK: setle +; CHECK: retl + +define i1 @test2(double %d) #0 { +entry: + %cmp = fcmp ult double %d, 0.000000e+00 + ret i1 %cmp +} +; CHECK-LABEL: test2: +; CHECK: calll __gedf2 +; CHECK: sets +; CHECK: retl + +define i1 @test3(double %d) #0 { +entry: + %cmp = fcmp ugt double %d, 0.000000e+00 + ret i1 %cmp +} +; CHECK-LABEL: test3: +; CHECK: calll __ledf2 +; CHECK: setg +; CHECK: retl + +define i1 @test4(double %d) #0 { +entry: + %cmp = fcmp uge double %d, 0.000000e+00 + ret i1 %cmp +} +; CHECK-LABEL: test4: +; CHECK: calll __ltdf2 +; CHECK: setns +; CHECK: retl + +define i1 @test5(double %d) #0 { +entry: + %cmp = fcmp ole double %d, 0.000000e+00 + ret i1 %cmp +} +; CHECK-LABEL: test5: +; CHECK: calll __ledf2 +; CHECK: setle +; CHECK: retl + +define i1 @test6(double %d) #0 { +entry: + %cmp = fcmp olt double %d, 0.000000e+00 + ret i1 %cmp +} +; CHECK-LABEL: test6: +; CHECK: calll __ltdf2 +; CHECK: sets +; CHECK: retl + +define i1 @test7(double %d) #0 { +entry: + %cmp = fcmp ogt double %d, 0.000000e+00 + ret i1 %cmp +} +; CHECK-LABEL: test7: +; CHECK: calll __gtdf2 +; CHECK: setg +; CHECK: retl + +define i1 @test8(double %d) #0 { +entry: + %cmp = fcmp oge double %d, 0.000000e+00 + ret i1 %cmp +} +; CHECK-LABEL: test8: +; CHECK: calll __gedf2 +; CHECK: setns +; CHECK: retl + +define i1 @test9(double %d) #0 { +entry: + %cmp = fcmp oeq double %d, 0.000000e+00 + ret i1 %cmp +} +; CHECK-LABEL: test9: +; CHECK: calll __eqdf2 +; CHECK: sete +; CHECK: retl + +define i1 @test10(double %d) #0 { +entry: + %cmp = fcmp ueq double %d, 0.000000e+00 + ret i1 %cmp +} +; CHECK-LABEL: test10: +; CHECK: calll __eqdf2 +; CHECK: sete +; CHECK: calll __unorddf2 +; CHECK: setne +; CHECK: retl + +define i1 @test11(double %d) #0 { +entry: + %cmp = fcmp one double %d, 0.000000e+00 + ret i1 %cmp +} +; CHECK-LABEL: test11: +; CHECK: calll __gtdf2 +; CHECK: setg +; CHECK: calll __ltdf2 +; CHECK: sets +; CHECK: retl + +define i1 @test12(double %d) #0 { +entry: + %cmp = fcmp une double %d, 0.000000e+00 + ret i1 %cmp +} +; CHECK-LABEL: test12: +; CHECK: calll __nedf2 +; CHECK: setne +; CHECK: retl + +attributes #0 = { "use-soft-float"="true" } diff --git a/test/CodeGen/X86/fpstack-debuginstr-kill.ll b/test/CodeGen/X86/fpstack-debuginstr-kill.ll index 34398414a76c..2ee67dc190bd 100644 --- a/test/CodeGen/X86/fpstack-debuginstr-kill.ll +++ b/test/CodeGen/X86/fpstack-debuginstr-kill.ll @@ -3,7 +3,7 @@ @g1 = global double 0.000000e+00, align 8 @g2 = global i32 0, align 4 -define void @_Z16fpuop_arithmeticjj(i32, i32) { +define void @_Z16fpuop_arithmeticjj(i32, i32) !dbg !4 { entry: switch i32 undef, label %sw.bb.i1921 [ ] @@ -43,27 +43,27 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!24, !25} -!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (http://llvm.org/git/clang 8444ae7cfeaefae031f8fedf0d1435ca3b14d90b) (http://llvm.org/git/llvm 886f0101a7d176543b831f5efb74c03427244a55)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !21, imports: !2) +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (http://llvm.org/git/clang 8444ae7cfeaefae031f8fedf0d1435ca3b14d90b) (http://llvm.org/git/llvm 886f0101a7d176543b831f5efb74c03427244a55)", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !21, imports: !2) !1 = !DIFile(filename: "fpu_ieee.cpp", directory: "x87stackifier") !2 = !{} !3 = !{!4} -!4 = !DISubprogram(name: "fpuop_arithmetic", linkageName: "_Z16fpuop_arithmeticjj", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 13, file: !5, scope: !6, type: !7, function: void (i32, i32)* @_Z16fpuop_arithmeticjj, variables: !10) +!4 = distinct !DISubprogram(name: "fpuop_arithmetic", linkageName: "_Z16fpuop_arithmeticjj", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 13, file: !5, scope: !6, type: !7, variables: !10) !5 = !DIFile(filename: "f1.cpp", directory: "x87stackifier") !6 = !DIFile(filename: "f1.cpp", directory: "x87stackifier") !7 = !DISubroutineType(types: !8) !8 = !{null, !9, !9} !9 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned) !10 = !{!11, !12, !13, !18, !20} -!11 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 11, arg: 1, scope: !4, file: !6, type: !9) -!12 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 11, arg: 2, scope: !4, file: !6, type: !9) -!13 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "x", line: 14, scope: !4, file: !6, type: !14) +!11 = !DILocalVariable(name: "", line: 11, arg: 1, scope: !4, file: !6, type: !9) +!12 = !DILocalVariable(name: "", line: 11, arg: 2, scope: !4, file: !6, type: !9) +!13 = !DILocalVariable(name: "x", line: 14, scope: !4, file: !6, type: !14) !14 = !DIDerivedType(tag: DW_TAG_typedef, name: "fpu_extended", line: 3, file: !5, baseType: !15) !15 = !DIDerivedType(tag: DW_TAG_typedef, name: "fpu_register", line: 2, file: !5, baseType: !16) !16 = !DIDerivedType(tag: DW_TAG_typedef, name: "uae_f64", line: 1, file: !5, baseType: !17) !17 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float) -!18 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "a", line: 15, scope: !4, file: !6, type: !19) +!18 = !DILocalVariable(name: "a", line: 15, scope: !4, file: !6, type: !19) !19 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) -!20 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "value", line: 16, scope: !4, file: !6, type: !14) +!20 = !DILocalVariable(name: "value", line: 16, scope: !4, file: !6, type: !14) !21 = !{!22, !23} !22 = !DIGlobalVariable(name: "g1", line: 5, isLocal: false, isDefinition: true, scope: null, file: !6, type: !14, variable: double* @g1) !23 = !DIGlobalVariable(name: "g2", line: 6, isLocal: false, isDefinition: true, scope: null, file: !6, type: !19, variable: i32* @g2) diff --git a/test/CodeGen/X86/frameescape.ll b/test/CodeGen/X86/frameescape.ll deleted file mode 100644 index 179a936304ba..000000000000 --- a/test/CodeGen/X86/frameescape.ll +++ /dev/null @@ -1,128 +0,0 @@ -; RUN: llc -mtriple=i686-windows-msvc < %s | FileCheck %s --check-prefix=X86 -; RUN: llc -mtriple=x86_64-windows-msvc < %s | FileCheck %s --check-prefix=X64 - -declare void @llvm.localescape(...) -declare i8* @llvm.frameaddress(i32) -declare i8* @llvm.localrecover(i8*, i8*, i32) -declare i32 @printf(i8*, ...) - -@str = internal constant [10 x i8] c"asdf: %d\0A\00" - -define void @print_framealloc_from_fp(i8* %fp) { - %a.i8 = call i8* @llvm.localrecover(i8* bitcast (void()* @alloc_func to i8*), i8* %fp, i32 0) - %a = bitcast i8* %a.i8 to i32* - %a.val = load i32, i32* %a - call i32 (i8*, ...) @printf(i8* getelementptr ([10 x i8], [10 x i8]* @str, i32 0, i32 0), i32 %a.val) - %b.i8 = call i8* @llvm.localrecover(i8* bitcast (void()* @alloc_func to i8*), i8* %fp, i32 1) - %b = bitcast i8* %b.i8 to i32* - %b.val = load i32, i32* %b - call i32 (i8*, ...) @printf(i8* getelementptr ([10 x i8], [10 x i8]* @str, i32 0, i32 0), i32 %b.val) - store i32 42, i32* %b - %b2 = getelementptr i32, i32* %b, i32 1 - %b2.val = load i32, i32* %b2 - call i32 (i8*, ...) @printf(i8* getelementptr ([10 x i8], [10 x i8]* @str, i32 0, i32 0), i32 %b2.val) - ret void -} - -; X64-LABEL: print_framealloc_from_fp: -; X64: movq %rcx, %[[parent_fp:[a-z]+]] -; X64: movl .Lalloc_func$frame_escape_0(%[[parent_fp]]), %edx -; X64: leaq {{.*}}(%rip), %[[str:[a-z]+]] -; X64: movq %[[str]], %rcx -; X64: callq printf -; X64: movl .Lalloc_func$frame_escape_1(%[[parent_fp]]), %edx -; X64: movq %[[str]], %rcx -; X64: callq printf -; X64: movl $42, .Lalloc_func$frame_escape_1(%[[parent_fp]]) -; X64: retq - -; X86-LABEL: print_framealloc_from_fp: -; X86: pushl %esi -; X86: subl $8, %esp -; X86: movl 16(%esp), %esi -; X86: movl Lalloc_func$frame_escape_0(%esi), %eax -; X86: movl %eax, 4(%esp) -; X86: movl $_str, (%esp) -; X86: calll _printf -; X86: movl Lalloc_func$frame_escape_1(%esi), %eax -; X86: movl %eax, 4(%esp) -; X86: movl $_str, (%esp) -; X86: calll _printf -; X86: movl $42, Lalloc_func$frame_escape_1(%esi) -; X86: movl $4, %eax -; X86: movl Lalloc_func$frame_escape_1(%esi,%eax), %eax -; X86: movl %eax, 4(%esp) -; X86: movl $_str, (%esp) -; X86: calll _printf -; X86: addl $8, %esp -; X86: popl %esi -; X86: retl - -define void @alloc_func() { - %a = alloca i32 - %b = alloca i32, i32 2 - call void (...) @llvm.localescape(i32* %a, i32* %b) - store i32 42, i32* %a - store i32 13, i32* %b - %fp = call i8* @llvm.frameaddress(i32 0) - call void @print_framealloc_from_fp(i8* %fp) - ret void -} - -; X64-LABEL: alloc_func: -; X64: subq $48, %rsp -; X64: .seh_stackalloc 48 -; X64: leaq 48(%rsp), %rbp -; X64: .seh_setframe 5, 48 -; X64: .Lalloc_func$frame_escape_0 = 44 -; X64: .Lalloc_func$frame_escape_1 = 36 -; X64: movl $42, -4(%rbp) -; X64: movl $13, -12(%rbp) -; X64: leaq -48(%rbp), %rcx -; X64: callq print_framealloc_from_fp -; X64: retq - -; X86-LABEL: alloc_func: -; X86: pushl %ebp -; X86: movl %esp, %ebp -; X86: subl $16, %esp -; X86: Lalloc_func$frame_escape_0 = -4 -; X86: Lalloc_func$frame_escape_1 = -12 -; X86: movl $42, -4(%ebp) -; X86: movl $13, -12(%ebp) -; X86: movl %ebp, (%esp) -; X86: calll _print_framealloc_from_fp -; X86: addl $16, %esp -; X86: popl %ebp -; X86: retl - -; Helper to make this a complete program so it can be compiled and tested. -define i32 @main() { - call void @alloc_func() - ret i32 0 -} - -define void @alloc_func_no_frameaddr() { - %a = alloca i32 - %b = alloca i32 - call void (...) @llvm.localescape(i32* %a, i32* %b) - store i32 42, i32* %a - store i32 13, i32* %b - call void @print_framealloc_from_fp(i8* null) - ret void -} - -; X64-LABEL: alloc_func_no_frameaddr: -; X64: subq $40, %rsp -; X64: .seh_stackalloc 40 -; X64: .seh_endprologue -; X64: .Lalloc_func_no_frameaddr$frame_escape_0 = 36 -; X64: .Lalloc_func_no_frameaddr$frame_escape_1 = 32 -; X64: movl $42, 36(%rsp) -; X64: movl $13, 32(%rsp) -; X64: xorl %ecx, %ecx -; X64: callq print_framealloc_from_fp -; X64: addq $40, %rsp -; X64: retq - -; X86-LABEL: alloc_func_no_frameaddr: diff --git a/test/CodeGen/X86/frem-msvc32.ll b/test/CodeGen/X86/frem-msvc32.ll new file mode 100644 index 000000000000..01144eb44de4 --- /dev/null +++ b/test/CodeGen/X86/frem-msvc32.ll @@ -0,0 +1,12 @@ +; Make sure that 32-bit FREM is promoted to 64-bit FREM on 32-bit MSVC. + +; MSVC does not have a 32-bit fmodf function, so it must be promoted to +; a 64-bit fmod rtlib call. +; RUN: llc -mtriple=i686-pc-windows-msvc -O0 < %s | FileCheck %s + +; CHECK: @do_frem32 +; CHECK: {{_fmod$}} +define float @do_frem32(float %a, float %b) { + %val = frem float %a, %b + ret float %val +} diff --git a/test/CodeGen/X86/funclet-layout.ll b/test/CodeGen/X86/funclet-layout.ll new file mode 100644 index 000000000000..0942645cf5a4 --- /dev/null +++ b/test/CodeGen/X86/funclet-layout.ll @@ -0,0 +1,158 @@ +; RUN: llc -mtriple=x86_64-windows-msvc < %s | FileCheck %s + +target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc" + +%eh.ThrowInfo = type { i32, i32, i32, i32 } +%rtti.TypeDescriptor2 = type { i8**, i8*, [3 x i8] } + +@"\01??_7type_info@@6B@" = external constant i8* +@"\01??_R0H@8" = internal global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".H\00" } + +define void @test1(i1 %B) personality i32 (...)* @__CxxFrameHandler3 { +entry: + invoke void @g() + to label %unreachable unwind label %catch.dispatch + +catch.dispatch: + %cs1 = catchswitch within none [label %catch] unwind to caller + +catch: + %cp = catchpad within %cs1 [i8* null, i32 64, i8* null] + br label %catch.loop + +catch.loop: + br i1 %B, label %catchret, label %catch.loop + +catchret: + catchret from %cp to label %try.cont + +try.cont: + ret void + +unreachable: + unreachable +} + +; CHECK-LABEL: test1: + +; The entry funclet contains %entry and %try.cont +; CHECK: # %entry +; CHECK: # %try.cont +; CHECK: retq + +; The catch funclet contains %catch and %catchret +; CHECK: # %catch{{$}} +; CHECK: # %catchret +; CHECK: retq + +declare void @g() + + +define i32 @test2(i1 %B) personality i32 (...)* @__CxxFrameHandler3 { +entry: + invoke void @_CxxThrowException(i8* null, %eh.ThrowInfo* null) #1 + to label %unreachable unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %cs1 = catchswitch within none [label %catch] unwind to caller + +catch: ; preds = %catch.dispatch + %0 = catchpad within %cs1 [i8* null, i32 64, i8* null] + invoke void @_CxxThrowException(i8* null, %eh.ThrowInfo* null) #1 ["funclet"(token %0)] + to label %unreachable unwind label %catch.dispatch.1 + +catch.dispatch.1: ; preds = %catch + %cs2 = catchswitch within %0 [label %catch.3] unwind to caller + +catch.3: ; preds = %catch.dispatch.1 + %1 = catchpad within %cs2 [i8* null, i32 64, i8* null] + catchret from %1 to label %try.cont + +try.cont: ; preds = %catch.3 + catchret from %0 to label %try.cont.5 + +try.cont.5: ; preds = %try.cont + ret i32 0 + +unreachable: ; preds = %catch, %entry + unreachable +} + +; CHECK-LABEL: test2: + +; The parent function contains %entry and %try.cont.5 +; CHECK: .seh_proc +; CHECK: # %entry +; CHECK: # %try.cont.5 +; CHECK: retq + +; The inner catch funclet contains %catch.3 +; CHECK: .seh_proc +; CHECK: # %catch.3{{$}} +; CHECK: retq + +; The outer catch funclet contains %catch +; CHECK: .seh_proc +; CHECK: # %catch{{$}} +; CHECK: callq _CxxThrowException +; CHECK: # %unreachable +; CHECK: ud2 + + +define void @test3(i1 %V) #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) { +entry: + invoke void @g() + to label %try.cont unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %cs1 = catchswitch within none [label %catch.2] unwind label %catch.dispatch.1 + +catch.2: ; preds = %catch.dispatch + %0 = catchpad within %cs1 [%rtti.TypeDescriptor2* @"\01??_R0H@8", i32 0, i8* null] + tail call void @exit(i32 0) #2 [ "funclet"(token %0) ] + unreachable + +catch.dispatch.1: ; preds = %catch.dispatch + %cs2 = catchswitch within none [label %catch] unwind to caller + +catch: ; preds = %catch.dispatch.1 + %1 = catchpad within %cs2 [i8* null, i32 64, i8* null] + tail call void @exit(i32 0) #2 [ "funclet"(token %1) ] + unreachable + +try.cont: ; preds = %entry + br i1 %V, label %exit_one, label %exit_two + +exit_one: + tail call void @exit(i32 0) + unreachable + +exit_two: + tail call void @exit(i32 0) + unreachable +} + +; CHECK-LABEL: test3: + +; The entry funclet contains %entry and %try.cont +; CHECK: # %entry +; CHECK: # %try.cont +; CHECK: callq exit +; CHECK-NOT: # exit_one +; CHECK-NOT: # exit_two +; CHECK: ud2 + +; The catch(...) funclet contains %catch.2 +; CHECK: # %catch.2{{$}} +; CHECK: callq exit +; CHECK: ud2 + +; The catch(int) funclet contains %catch +; CHECK: # %catch{{$}} +; CHECK: callq exit +; CHECK: ud2 + +declare void @exit(i32) noreturn nounwind +declare void @_CxxThrowException(i8*, %eh.ThrowInfo*) +declare i32 @__CxxFrameHandler3(...) diff --git a/test/CodeGen/X86/function-alias.ll b/test/CodeGen/X86/function-alias.ll new file mode 100644 index 000000000000..d68d75d5578a --- /dev/null +++ b/test/CodeGen/X86/function-alias.ll @@ -0,0 +1,12 @@ +; RUN: llc < %s | FileCheck %s +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; "data" constant +@0 = private constant <{ i8, i8 }> <{i8 15, i8 11}>, section ".text" + +; function-typed alias +@ud2 = alias void (), bitcast (<{ i8, i8 }>* @0 to void ()*) + +; Check that "ud2" is emitted as a function symbol. +; CHECK: .type{{.*}}ud2,@function diff --git a/test/CodeGen/X86/gcc_except_table.ll b/test/CodeGen/X86/gcc_except_table.ll index 82064c2a3907..92ea539bcf77 100644 --- a/test/CodeGen/X86/gcc_except_table.ll +++ b/test/CodeGen/X86/gcc_except_table.ll @@ -18,9 +18,9 @@ define i32 @main() uwtable optsize ssp personality i8* bitcast (i32 (...)* @__gx ; MINGW64: .seh_setframe 5, 32 ; MINGW64: callq _Unwind_Resume ; MINGW64: .seh_handlerdata +; MINGW64: .seh_endproc ; MINGW64: GCC_except_table0: ; MINGW64: Lexception0: -; MINGW64: .seh_endproc ; MINGW32: .cfi_startproc ; MINGW32: .cfi_personality 0, ___gxx_personality_v0 diff --git a/test/CodeGen/X86/global-sections.ll b/test/CodeGen/X86/global-sections.ll index 82547a606742..92440f2b3316 100644 --- a/test/CodeGen/X86/global-sections.ll +++ b/test/CodeGen/X86/global-sections.ll @@ -117,7 +117,7 @@ bb7: ; TODO: linux drops this into .rodata, we drop it into ".gnu.linkonce.r.G2" -; DARWIN: .section __TEXT,__const_coal,coalesced +; DARWIN: .section __TEXT,__const{{$}} ; DARWIN: _G2: ; DARWIN: .long 42 @@ -176,7 +176,6 @@ bb7: ; LINUX: .weak "foo bar" ; LINUX: "foo bar": -; DARWIN: .section __DATA,__datacoal_nt,coalesced ; DARWIN: .globl "_foo bar" ; DARWIN: .weak_definition "_foo bar" ; DARWIN: "_foo bar": @@ -190,7 +189,7 @@ bb7: ; LINUX: .byte 1 ; LINUX: .size G6, 1 -; DARWIN: .section __TEXT,__const_coal,coalesced +; DARWIN: .section __TEXT,__const{{$}} ; DARWIN: .globl _G6 ; DARWIN: .weak_definition _G6 ; DARWIN:_G6: @@ -239,7 +238,7 @@ bb7: @G10 = weak global [100 x i32] zeroinitializer, align 32 ; <[100 x i32]*> [#uses=0] -; DARWIN: .section __DATA,__datacoal_nt,coalesced +; DARWIN: .section __DATA,__data{{$}} ; DARWIN: .globl _G10 ; DARWIN: .weak_definition _G10 ; DARWIN: .align 5 diff --git a/test/CodeGen/X86/h-register-store.ll b/test/CodeGen/X86/h-register-store.ll index 0adb2b148c39..0e6a0236d2c3 100644 --- a/test/CodeGen/X86/h-register-store.ll +++ b/test/CodeGen/X86/h-register-store.ll @@ -7,6 +7,15 @@ ; X64-NEXT: movb %ah, (%rsi) ; X64-NOT: mov +; RUN: llc < %s -mtriple=x86_64-linux-gnux32 | FileCheck %s -check-prefix=X32 +; X32: mov +; X32-NEXT: movb %ah, (%esi) +; X32: mov +; X32-NEXT: movb %ah, (%esi) +; X32: mov +; X32-NEXT: movb %ah, (%esi) +; X32-NOT: mov + ; RUN: llc < %s -mtriple=x86_64-win32 | FileCheck %s -check-prefix=W64 ; W64-NOT: mov ; W64: movb %ch, (%rdx) @@ -16,14 +25,14 @@ ; W64: movb %ch, (%rdx) ; W64-NOT: mov -; RUN: llc < %s -march=x86 | FileCheck %s -check-prefix=X32 -; X32-NOT: mov -; X32: movb %ah, (%e -; X32-NOT: mov -; X32: movb %ah, (%e -; X32-NOT: mov -; X32: movb %ah, (%e -; X32-NOT: mov +; RUN: llc < %s -march=x86 | FileCheck %s -check-prefix=X86 +; X86-NOT: mov +; X86: movb %ah, (%e +; X86-NOT: mov +; X86: movb %ah, (%e +; X86-NOT: mov +; X86: movb %ah, (%e +; X86-NOT: mov ; Use h-register extract and store. diff --git a/test/CodeGen/X86/h-registers-0.ll b/test/CodeGen/X86/h-registers-0.ll index 6a5ccaa1e76f..9b72916ea743 100644 --- a/test/CodeGen/X86/h-registers-0.ll +++ b/test/CodeGen/X86/h-registers-0.ll @@ -1,4 +1,5 @@ ; RUN: llc < %s -mattr=-bmi -mtriple=x86_64-linux | FileCheck %s -check-prefix=X86-64 +; RUN: llc < %s -mattr=-bmi -mtriple=x86_64-linux-gnux32 | FileCheck %s -check-prefix=X86-64 ; RUN: llc < %s -mattr=-bmi -mtriple=x86_64-win32 | FileCheck %s -check-prefix=WIN64 ; RUN: llc < %s -mattr=-bmi -march=x86 | FileCheck %s -check-prefix=X86-32 diff --git a/test/CodeGen/X86/h-registers-1.ll b/test/CodeGen/X86/h-registers-1.ll index 7254325a9265..469d5517b40b 100644 --- a/test/CodeGen/X86/h-registers-1.ll +++ b/test/CodeGen/X86/h-registers-1.ll @@ -1,4 +1,5 @@ ; RUN: llc -mattr=-bmi < %s -mtriple=x86_64-linux | FileCheck %s +; RUN: llc -mattr=-bmi < %s -mtriple=x86_64-linux-gnux32 | FileCheck %s ; LLVM creates virtual registers for values live across blocks ; based on the type of the value. Make sure that the extracts diff --git a/test/CodeGen/X86/h-registers-3.ll b/test/CodeGen/X86/h-registers-3.ll index 29d0c280c4fb..58b02b7df21f 100644 --- a/test/CodeGen/X86/h-registers-3.ll +++ b/test/CodeGen/X86/h-registers-3.ll @@ -1,5 +1,6 @@ ; RUN: llc < %s -march=x86 | grep mov | count 1 ; RUN: llc < %s -march=x86-64 | grep mov | count 1 +; RUN: llc < %s -mtriple=x86_64-linux-gnux32 | grep mov | count 1 define zeroext i8 @foo() nounwind ssp { entry: diff --git a/test/CodeGen/X86/half.ll b/test/CodeGen/X86/half.ll index 8a726370f19a..3b2518e28f58 100644 --- a/test/CodeGen/X86/half.ll +++ b/test/CodeGen/X86/half.ll @@ -77,7 +77,7 @@ define i64 @test_fptosi_i64(half* %p) #0 { ; CHECK-LIBCALL-NEXT: movzwl (%rdi), %edi ; CHECK-LIBCALL-NEXT: callq __gnu_h2f_ieee ; CHECK-LIBCALL-NEXT: cvttss2si %xmm0, %rax -; CHECK-LIBCALL-NEXT: popq %rdx +; CHECK-LIBCALL-NEXT: popq %rcx ; CHECK-LIBCALL-NEXT: retq ; CHECK-F16C-NEXT: movswl (%rdi), [[REG0:%[a-z0-9]+]] @@ -127,7 +127,7 @@ define i64 @test_fptoui_i64(half* %p) #0 { ; CHECK-LIBCALL-NEXT: cvttss2si %xmm0, [[REG5:%[a-z0-9]+]] ; CHECK-LIBCALL-NEXT: ucomiss [[REG1]], %xmm0 ; CHECK-LIBCALL-NEXT: cmovaeq [[REG4]], [[REG5]] -; CHECK-LIBCALL-NEXT: popq %rdx +; CHECK-LIBCALL-NEXT: popq %rcx ; CHECK-LIBCALL-NEXT: retq ; CHECK-F16C-NEXT: movswl (%rdi), [[REG0:%[a-z0-9]+]] diff --git a/test/CodeGen/X86/hhvm-cc.ll b/test/CodeGen/X86/hhvm-cc.ll new file mode 100644 index 000000000000..3b729ed72f1c --- /dev/null +++ b/test/CodeGen/X86/hhvm-cc.ll @@ -0,0 +1,241 @@ +; RUN: llc < %s | FileCheck %s + +target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +declare hhvmcc i64 @bar(i64, i64, i64) nounwind + +; Simply check we can modify %rbx and %rbp before returning via call to bar. +define hhvmcc i64 @foo(i64 %a, i64 %b, i64 %c) nounwind { +entry: +; CHECK-LABEL: foo: +; CHECK-DAG: movl $1, %ebx +; CHECK-DAG: movl $3, %ebp +; CHECK: jmp bar + %ret = musttail call hhvmcc i64 @bar(i64 1, i64 %b, i64 3) + ret i64 %ret +} + +; Check that we can read and modify %rbx returned from PHP function. +define hhvmcc i64 @mod_return(i64 %a, i64 %b, i64 %c) nounwind { +entry: +; CHECK-LABEL: mod_return: +; CHECK-NEXT: {{^#.*}} +; CHECK-NEXT: callq bar +; CHECK-NEXT: incq %rbx + %tmp = call hhvmcc i64 @bar(i64 %a, i64 %b, i64 %c) + %retval = add i64 %tmp, 1 + ret i64 %retval +} + +%rettype = type { i64, i64, i64, i64, i64, i64, i64, + i64, i64, i64, i64, i64, i64, i64 +} + +; Check that we can return up to 14 64-bit args in registers. +define hhvmcc %rettype @return_all(i64 %a, i64 %b, i64 %c) nounwind { +entry: +; CHECK-LABEL: return_all: +; CHECK-DAG: movl $1, %ebx +; CHECK-DAG: movl $2, %ebp +; CHECK-DAG: movl $3, %edi +; CHECK-DAG: movl $4, %esi +; CHECK-DAG: movl $5, %edx +; CHECK-DAG: movl $6, %ecx +; CHECK-DAG: movl $7, %r8 +; CHECK-DAG: movl $8, %r9 +; CHECK-DAG: movl $9, %eax +; CHECK-DAG: movl $10, %r10 +; CHECK-DAG: movl $11, %r11 +; CHECK-DAG: movl $12, %r13 +; CHECK-DAG: movl $13, %r14 +; CHECK-DAG: movl $14, %r15 +; CHECK: retq + %r1 = insertvalue %rettype zeroinitializer, i64 1, 0 + %r2 = insertvalue %rettype %r1, i64 2, 1 + %r3 = insertvalue %rettype %r2, i64 3, 2 + %r4 = insertvalue %rettype %r3, i64 4, 3 + %r5 = insertvalue %rettype %r4, i64 5, 4 + %r6 = insertvalue %rettype %r5, i64 6, 5 + %r7 = insertvalue %rettype %r6, i64 7, 6 + %r8 = insertvalue %rettype %r7, i64 8, 7 + %r9 = insertvalue %rettype %r8, i64 9, 8 + %r10 = insertvalue %rettype %r9, i64 10, 9 + %r11 = insertvalue %rettype %r10, i64 11, 10 + %r12 = insertvalue %rettype %r11, i64 12, 11 + %r13 = insertvalue %rettype %r12, i64 13, 12 + %r14 = insertvalue %rettype %r13, i64 14, 13 + ret %rettype %r14 +} + +declare hhvmcc void @return_all_tc(i64, i64, i64, i64, i64, i64, i64, i64, + i64, i64, i64, i64, i64, i64, i64) + +; Check that we can return up to 14 64-bit args in registers via tail call. +define hhvmcc void @test_return_all_tc(i64 %a, i64 %b, i64 %c) nounwind { +entry: +; CHECK-LABEL: test_return_all_tc: +; CHECK-NEXT: {{^#.*}} +; CHECK-DAG: movl $1, %ebx +; CHECK-DAG: movl $3, %ebp +; CHECK-DAG: movl $4, %r15 +; CHECK-DAG: movl $5, %edi +; CHECK-DAG: movl $6, %esi +; CHECK-DAG: movl $7, %edx +; CHECK-DAG: movl $8, %ecx +; CHECK-DAG: movl $9, %r8 +; CHECK-DAG: movl $10, %r9 +; CHECK-DAG: movl $11, %eax +; CHECK-DAG: movl $12, %r10 +; CHECK-DAG: movl $13, %r11 +; CHECK-DAG: movl $14, %r13 +; CHECK-DAG: movl $15, %r14 +; CHECK: jmp return_all_tc + tail call hhvmcc void @return_all_tc( + i64 1, i64 %b, i64 3, i64 4, i64 5, i64 6, i64 7, + i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15) + ret void +} + +declare hhvmcc {i64, i64} @php_short(i64, i64, i64, i64) + +define hhvmcc i64 @test_php_short(i64 %a, i64 %b, i64 %c) nounwind { +entry: +; CHECK-LABEL: test_php_short: +; CHECK-NEXT: {{^#.*}} +; CHECK-NEXT: movl $42, %r15 +; CHECK-NEXT: callq php_short +; CHECK-NEXT: leaq (%rbp,%r12), %rbx +; CHECK-NEXT: retq + %pair = call hhvmcc {i64, i64} @php_short(i64 %a, i64 %b, i64 %c, i64 42) + %fp = extractvalue {i64, i64} %pair, 1 + %rv = add i64 %fp, %b + ret i64 %rv +} + +declare hhvmcc %rettype @php_all(i64, i64, i64, i64, i64, i64, i64, + i64, i64, i64, i64, i64, i64, i64, i64) + +; Check that we can pass 15 arguments in registers. +; Also check that %r12 (2nd arg) is not spilled. +define hhvmcc i64 @test_php_all(i64 %a, i64 %b, i64 %c) nounwind { +entry: +; CHECK-LABEL: test_php_all: +; CHECK-NEXT: {{^#.*}} +; CHECK-NOT: sub +; CHECK-NOT: sub +; CHECK-DAG: movl $1, %ebx +; CHECK-DAG: movl $3, %ebp +; CHECK-DAG: movl $4, %r15 +; CHECK-DAG: movl $5, %edi +; CHECK-DAG: movl $6, %esi +; CHECK-DAG: movl $7, %edx +; CHECK-DAG: movl $8, %ecx +; CHECK-DAG: movl $9, %r8 +; CHECK-DAG: movl $10, %r9 +; CHECK-DAG: movl $11, %eax +; CHECK-DAG: movl $12, %r10 +; CHECK-DAG: movl $13, %r11 +; CHECK-DAG: movl $14, %r13 +; CHECK-DAG: movl $15, %r14 +; CHECK: callq php_all + %pair = call hhvmcc %rettype @php_all( + i64 1, i64 %b, i64 3, i64 4, i64 5, i64 6, i64 7, + i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15) + %fp = extractvalue %rettype %pair, 1 + %rv = add i64 %fp, %b + ret i64 %rv +} + +declare hhvmcc void @svcreq(i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, + i64, i64) + +define hhvmcc void @test_svcreq(i64 %a, i64 %b, i64 %c) nounwind { +entry: +; CHECK-LABEL: test_svcreq: +; CHECK-DAG: movl $42, %r10 +; CHECK-DAG: movl $1, %edi +; CHECK-DAG: movl $2, %esi +; CHECK-DAG: movl $3, %edx +; CHECK-DAG: movl $4, %ecx +; CHECK-DAG: movl $5, %r8 +; CHECK-DAG: movl $6, %r9 +; CHECK: jmp svcreq + tail call hhvmcc void @svcreq(i64 %a, i64 %b, i64 %c, i64 undef, i64 1, + i64 2, i64 3, i64 4, i64 5, i64 6, i64 undef, + i64 42) + ret void +} + +declare hhvm_ccc void @helper_short(i64, i64, i64, i64, i64, i64, i64) + +; Pass all arguments in registers and check that we don't adjust stack +; for the call. +define hhvmcc void @test_helper_short(i64 %a, i64 %b, i64 %c) nounwind { +entry: +; CHECK-LABEL: test_helper_short: +; CHECK-NOT: push +; CHECK-NOT: sub +; CHECK-DAG: movl $1, %edi +; CHECK-DAG: movl $2, %esi +; CHECK-DAG: movl $3, %edx +; CHECK-DAG: movl $4, %ecx +; CHECK-DAG: movl $5, %r8 +; CHECK-DAG: movl $6, %r9 +; CHECK: callq helper_short + call hhvm_ccc void @helper_short(i64 %c, i64 1, i64 2, i64 3, i64 4, + i64 5, i64 6) + ret void +} + +declare hhvm_ccc void @helper(i64, i64, i64, i64, i64, i64, i64, i64, i64, i64) + +define hhvmcc void @test_helper(i64 %a, i64 %b, i64 %c) nounwind { +entry: +; CHECK-LABEL: test_helper: +; CHECK-DAG: movl $1, %edi +; CHECK-DAG: movl $2, %esi +; CHECK-DAG: movl $3, %edx +; CHECK-DAG: movl $4, %ecx +; CHECK-DAG: movl $5, %r8 +; CHECK-DAG: movl $6, %r9 +; CHECK: callq helper + call hhvm_ccc void @helper(i64 %c, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, + i64 7, i64 8, i64 9) + ret void +} + +; When we enter function with HHVM calling convention, the stack is aligned +; at 16 bytes. This means we align objects on the stack differently and +; adjust the stack differently for calls. +declare hhvm_ccc void @stack_helper(i64, i64, i64) +declare hhvm_ccc void @stack_helper2(<2 x double>, i64) + +define hhvmcc void @test_stack_helper(i64 %a, i64 %b, i64 %c) nounwind { +entry: +; CHECK-LABEL: test_stack_helper: +; CHECK-NOT: push +; CHECK: subq $32, %rsp +; CHECK: movaps 16(%rsp), %xmm0 +; CHECK: callq stack_helper2 + %t1 = alloca <2 x double>, align 16 + %t2 = alloca i64, align 8 + %t3 = alloca i64, align 8 + %load3 = load i64, i64 *%t3 + call hhvm_ccc void @stack_helper(i64 %c, i64 %load3, i64 42) + %load = load <2 x double>, <2 x double> *%t1 + %load2 = load i64, i64 *%t2 + call hhvm_ccc void @stack_helper2(<2 x double> %load, i64 %load2) + ret void +} + +; Check that we are not adjusting the stack before calling the helper. +define hhvmcc void @test_stack_helper2(i64 %a, i64 %b, i64 %c) nounwind { +entry: +; CHECK-LABEL: test_stack_helper2: +; CHECK-NOT: push +; CHECK-NOT: subq + call hhvm_ccc void @stack_helper(i64 %c, i64 7, i64 42) + ret void +} + diff --git a/test/CodeGen/X86/i386-shrink-wrapping.ll b/test/CodeGen/X86/i386-shrink-wrapping.ll new file mode 100644 index 000000000000..748c397143c5 --- /dev/null +++ b/test/CodeGen/X86/i386-shrink-wrapping.ll @@ -0,0 +1,113 @@ +; RUN: llc %s -o - -enable-shrink-wrap=true | FileCheck %s --check-prefix=CHECK --check-prefix=ENABLE +; RUN: llc %s -o - -enable-shrink-wrap=false | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLE +target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128" +target triple = "i386-apple-macosx" + +@a = common global i32 0, align 4 +@d = internal unnamed_addr global i1 false +@b = common global i32 0, align 4 +@e = common global i8 0, align 1 +@f = common global i8 0, align 1 +@c = common global i32 0, align 4 +@.str = private unnamed_addr constant [4 x i8] c"%d\0A\00", align 1 + + +; Check that we are clobbering the flags when they are live-in of the +; prologue block and the prologue needs to adjust the stack. +; PR25607. +; +; CHECK-LABEL: eflagsLiveInPrologue: +; +; DISABLE: pushl +; DISABLE-NEXT: pushl +; DISABLE-NEXT: subl $20, %esp +; +; CHECK: movl L_a$non_lazy_ptr, [[A:%[a-z]+]] +; CHECK-NEXT: cmpl $0, ([[A]]) +; CHECK-NEXT: je [[PREHEADER_LABEL:LBB[0-9_]+]] +; +; CHECK: movb $1, _d +; +; CHECK: [[PREHEADER_LABEL]]: +; CHECK-NEXT: movl L_b$non_lazy_ptr, [[B:%[a-z]+]] +; CHECK-NEXT: movl ([[B]]), [[TMP1:%[a-z]+]] +; CHECK-NEXT: testl [[TMP1]], [[TMP1]] +; CHECK-NEXT: je [[FOREND_LABEL:LBB[0-9_]+]] +; +; Skip the loop. +; [...] +; +; The for.end block is split to accomadate the different selects. +; We are interested in the one with the call, so skip until the branch. +; CHECK: [[FOREND_LABEL]]: +; CHECK-NEXT: movb _d, [[D:%[a-z]+]] +; [...] +; CHECK: jne [[CALL_LABEL:LBB[0-9_]+]] +; +; CHECK: movb $6, [[D]] +; +; CHECK: [[CALL_LABEL]] +; +; ENABLE-NEXT: pushl +; ENABLE-NEXT: pushl +; We must not use sub here otherwise we will clobber the eflags. +; ENABLE-NEXT: leal -20(%esp), %esp +; +; CHECK-NEXT: L_e$non_lazy_ptr, [[E:%[a-z]+]] +; CHECK-NEXT: movb [[D]], ([[E]]) +; CHECK-NEXT: L_f$non_lazy_ptr, [[F:%[a-z]+]] +; CHECK-NEXT: movsbl ([[F]]), [[CONV:%[a-z]+]] +; CHECK-NEXT: movl $6, [[CONV:%[a-z]+]] +; The eflags is used in the next instruction. +; If that instruction disappear, we are not exercising the bug +; anymore. +; CHECK-NEXT: cmovnel {{%[a-z]+}}, [[CONV]] +; +; Skip all the crust of vaarg lowering. +; CHECK: calll L_varfunc$stub +; Set the return value to 0. +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: addl $20, %esp +; CHECK-NEXT: popl +; CHECK-NEXT: popl +; CHECK-NEXT: retl +define i32 @eflagsLiveInPrologue() #0 { +entry: + %tmp = load i32, i32* @a, align 4 + %tobool = icmp eq i32 %tmp, 0 + br i1 %tobool, label %for.cond.preheader, label %if.then + +if.then: ; preds = %entry + store i1 true, i1* @d, align 1 + br label %for.cond.preheader + +for.cond.preheader: ; preds = %if.then, %entry + %tmp1 = load i32, i32* @b, align 4 + %tobool14 = icmp eq i32 %tmp1, 0 + br i1 %tobool14, label %for.end, label %for.body.preheader + +for.body.preheader: ; preds = %for.cond.preheader + br label %for.body + +for.body: ; preds = %for.body, %for.body.preheader + br label %for.body + +for.end: ; preds = %for.cond.preheader + %.b3 = load i1, i1* @d, align 1 + %tmp2 = select i1 %.b3, i8 0, i8 6 + store i8 %tmp2, i8* @e, align 1 + %tmp3 = load i8, i8* @f, align 1 + %conv = sext i8 %tmp3 to i32 + %add = add nsw i32 %conv, 1 + %rem = srem i32 %tmp1, %add + store i32 %rem, i32* @c, align 4 + %conv2 = select i1 %.b3, i32 0, i32 6 + %call = tail call i32 (i8*, ...) @varfunc(i8* nonnull getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), i32 %conv2) #1 + ret i32 0 +} + +; Function Attrs: nounwind +declare i32 @varfunc(i8* nocapture readonly, ...) #0 + +attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { nounwind } diff --git a/test/CodeGen/X86/immediate_merging.ll b/test/CodeGen/X86/immediate_merging.ll new file mode 100644 index 000000000000..8aef9c279b31 --- /dev/null +++ b/test/CodeGen/X86/immediate_merging.ll @@ -0,0 +1,82 @@ +; RUN: llc -o - -mtriple=i386-unknown-linux-gnu < %s | FileCheck %s +; RUN: llc -o - -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s + +@a = common global i32 0, align 4 +@b = common global i32 0, align 4 +@c = common global i32 0, align 4 +@e = common global i32 0, align 4 +@x = common global i32 0, align 4 +@f = common global i32 0, align 4 +@h = common global i32 0, align 4 +@i = common global i32 0, align 4 + +; Test -Os to make sure immediates with multiple users don't get pulled in to +; instructions. +define i32 @foo() optsize { +; CHECK-LABEL: foo: +; CHECK: movl $1234, [[R1:%[a-z]+]] +; CHECK-NOT: movl $1234, a +; CHECK-NOT: movl $1234, b +; CHECK-NOT: movl $12, c +; CHECK-NOT: cmpl $12, e +; CHECK: movl [[R1]], a +; CHECK: movl [[R1]], b + +entry: + store i32 1234, i32* @a + store i32 1234, i32* @b + store i32 12, i32* @c + %0 = load i32, i32* @e + %cmp = icmp eq i32 %0, 12 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + store i32 1, i32* @x + br label %if.end + +; New block.. Make sure 1234 isn't live across basic blocks from before. +; CHECK: movl $1234, f +; CHECK: movl $555, [[R3:%[a-z]+]] +; CHECK-NOT: movl $555, h +; CHECK-NOT: addl $555, i +; CHECK: movl [[R3]], h +; CHECK: addl [[R3]], i + +if.end: ; preds = %if.then, %entry + store i32 1234, i32* @f + store i32 555, i32* @h + %1 = load i32, i32* @i + %add1 = add nsw i32 %1, 555 + store i32 %add1, i32* @i + ret i32 0 +} + +; Test -O2 to make sure that all immediates get pulled in to their users. +define i32 @foo2() { +; CHECK-LABEL: foo2: +; CHECK: movl $1234, a +; CHECK: movl $1234, b + +entry: + store i32 1234, i32* @a + store i32 1234, i32* @b + + ret i32 0 +} + +declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i32, i1) #1 + +@AA = common global [100 x i8] zeroinitializer, align 1 + +; memset gets lowered in DAG. Constant merging should hoist all the +; immediates used to store to the individual memory locations. Make +; sure we don't directly store the immediates. +define void @foomemset() optsize { +; CHECK-LABEL: foomemset: +; CHECK-NOT: movl ${{.*}}, AA +; CHECK: mov{{l|q}} %{{e|r}}ax, AA + +entry: + call void @llvm.memset.p0i8.i32(i8* getelementptr inbounds ([100 x i8], [100 x i8]* @AA, i32 0, i32 0), i8 33, i32 24, i32 1, i1 false) + ret void +} diff --git a/test/CodeGen/X86/implicit-null-check.ll b/test/CodeGen/X86/implicit-null-check.ll index fd7a902eefc1..8b905f5d23b6 100644 --- a/test/CodeGen/X86/implicit-null-check.ll +++ b/test/CodeGen/X86/implicit-null-check.ll @@ -101,6 +101,40 @@ define i32 @imp_null_check_hoist_over_unrelated_load(i32* %x, i32* %y, i32* %z) ret i32 %t1 } +define i32 @imp_null_check_via_mem_comparision(i32* %x, i32 %val) { +; CHECK-LABEL: _imp_null_check_via_mem_comparision +; CHECK: Ltmp9: +; CHECK: cmpl %esi, 4(%rdi) +; CHECK: jge LBB4_2 +; CHECK: movl $100, %eax +; CHECK: retq +; CHECK: Ltmp8: +; CHECK: movl $42, %eax +; CHECK: retq +; CHECK: LBB4_2: +; CHECK: movl $200, %eax +; CHECK: retq + + entry: + %c = icmp eq i32* %x, null + br i1 %c, label %is_null, label %not_null, !make.implicit !0 + + is_null: + ret i32 42 + + not_null: + %x.loc = getelementptr i32, i32* %x, i32 1 + %t = load i32, i32* %x.loc + %m = icmp slt i32 %t, %val + br i1 %m, label %ret_100, label %ret_200 + + ret_100: + ret i32 100 + + ret_200: + ret i32 200 +} + !0 = !{} ; CHECK-LABEL: __LLVM_FaultMaps: @@ -113,7 +147,7 @@ define i32 @imp_null_check_hoist_over_unrelated_load(i32* %x, i32* %y, i32* %z) ; CHECK-NEXT: .short 0 ; # functions: -; CHECK-NEXT: .long 4 +; CHECK-NEXT: .long 5 ; FunctionAddr: ; CHECK-NEXT: .quad _imp_null_check_add_result @@ -167,9 +201,22 @@ define i32 @imp_null_check_hoist_over_unrelated_load(i32* %x, i32* %y, i32* %z) ; Fault[0].HandlerOffset: ; CHECK-NEXT: .long Ltmp0-_imp_null_check_load +; FunctionAddr: +; CHECK-NEXT: .quad _imp_null_check_via_mem_comparision +; NumFaultingPCs +; CHECK-NEXT: .long 1 +; Reserved: +; CHECK-NEXT: .long 0 +; Fault[0].Type: +; CHECK-NEXT: .long 1 +; Fault[0].FaultOffset: +; CHECK-NEXT: .long Ltmp9-_imp_null_check_via_mem_comparision +; Fault[0].HandlerOffset: +; CHECK-NEXT: .long Ltmp8-_imp_null_check_via_mem_comparision + ; OBJDUMP: FaultMap table: ; OBJDUMP-NEXT: Version: 0x1 -; OBJDUMP-NEXT: NumFunctions: 4 +; OBJDUMP-NEXT: NumFunctions: 5 ; OBJDUMP-NEXT: FunctionAddress: 0x000000, NumFaultingPCs: 1 ; OBJDUMP-NEXT: Fault kind: FaultingLoad, faulting PC offset: 0, handling PC offset: 5 ; OBJDUMP-NEXT: FunctionAddress: 0x000000, NumFaultingPCs: 1 diff --git a/test/CodeGen/X86/imul.ll b/test/CodeGen/X86/imul.ll index c64b4e302b92..9d4d19332dbb 100644 --- a/test/CodeGen/X86/imul.ll +++ b/test/CodeGen/X86/imul.ll @@ -108,3 +108,66 @@ define i64 @mul40_64(i64 %A) { %mul = mul i64 %A, 40 ret i64 %mul } + +define i32 @mul4_32_minsize(i32 %A) minsize { +; X64-LABEL: mul4_32_minsize: +; X64: leal +; X86-LABEL: mul4_32_minsize: +; X86: shll + %mul = mul i32 %A, 4 + ret i32 %mul +} + +define i32 @mul40_32_minsize(i32 %A) minsize { +; X64-LABEL: mul40_32_minsize: +; X64: imull +; X86-LABEL: mul40_32_minsize: +; X86: imull + %mul = mul i32 %A, 40 + ret i32 %mul +} + +define i32 @mul33_32(i32 %A) { +; X64-LABEL: mul33_32: +; X64: shll +; X64-NEXT: leal +; X86-LABEL: mul33_32: +; X86: shll +; X86-NEXT: addl + %mul = mul i32 %A, 33 + ret i32 %mul +} + +define i32 @mul31_32(i32 %A) { +; X64-LABEL: mul31_32: +; X64: shll +; X64-NEXT: subl +; X86-LABEL: mul31_32: +; X86: shll +; X86-NEXT: subl + %mul = mul i32 %A, 31 + ret i32 %mul +} + +define i32 @mul0_32(i32 %A) { +; X64-LABEL: mul0_32: +; X64: xorl %eax, %eax + %mul = mul i32 %A, 0 + ret i32 %mul +} + +define i32 @mul4294967295_32(i32 %A) { +; X64-LABEL: mul4294967295_32: +; X64: negl %edi +; X64-NEXT: movl %edi, %eax + %mul = mul i32 %A, 4294967295 + ret i32 %mul +} + +define i64 @mul18446744073709551615_64(i64 %A) { +; X64-LABEL: mul18446744073709551615_64: +; X64: negq %rdi +; X64-NEXT: movq %rdi, %rax + %mul = mul i64 %A, 18446744073709551615 + ret i64 %mul +} diff --git a/test/CodeGen/X86/inalloca-stdcall.ll b/test/CodeGen/X86/inalloca-stdcall.ll index e5f6ea70e9cb..4f7e4092a99c 100644 --- a/test/CodeGen/X86/inalloca-stdcall.ll +++ b/test/CodeGen/X86/inalloca-stdcall.ll @@ -14,8 +14,9 @@ define void @g() { %f2 = getelementptr %Foo, %Foo* %b, i32 0, i32 1 store i32 13, i32* %f1 store i32 42, i32* %f2 -; CHECK: movl $13, (%esp) -; CHECK: movl $42, 4(%esp) +; CHECK: movl %esp, %eax +; CHECK: movl $13, (%eax) +; CHECK: movl $42, 4(%eax) call x86_stdcallcc void @f(%Foo* inalloca %b) ; CHECK: calll _f@8 ; CHECK-NOT: %esp diff --git a/test/CodeGen/X86/inalloca.ll b/test/CodeGen/X86/inalloca.ll index 904366219ab7..e523c945a69f 100644 --- a/test/CodeGen/X86/inalloca.ll +++ b/test/CodeGen/X86/inalloca.ll @@ -14,8 +14,9 @@ entry: %f2 = getelementptr %Foo, %Foo* %b, i32 0, i32 1 store i32 13, i32* %f1 store i32 42, i32* %f2 -; CHECK: movl $13, (%esp) -; CHECK: movl $42, 4(%esp) +; CHECK: movl %esp, %eax +; CHECK: movl $13, (%eax) +; CHECK: movl $42, 4(%eax) call void @f(%Foo* inalloca %b) ; CHECK: calll _f ret void @@ -33,8 +34,9 @@ entry: %f2 = getelementptr %Foo, %Foo* %b, i32 0, i32 1 store i32 13, i32* %f1 store i32 42, i32* %f2 -; CHECK: movl $13, (%esp) -; CHECK: movl $42, 4(%esp) +; CHECK: movl %esp, %eax +; CHECK: movl $13, (%eax) +; CHECK: movl $42, 4(%eax) call void @inreg_with_inalloca(i32 inreg 1, %Foo* inalloca %b) ; CHECK: movl $1, %eax ; CHECK: calll _inreg_with_inalloca @@ -53,8 +55,9 @@ entry: %f2 = getelementptr %Foo, %Foo* %b, i32 0, i32 1 store i32 13, i32* %f1 store i32 42, i32* %f2 -; CHECK-DAG: movl $13, (%esp) -; CHECK-DAG: movl $42, 4(%esp) +; CHECK: movl %esp, %eax +; CHECK-DAG: movl $13, (%eax) +; CHECK-DAG: movl $42, 4(%eax) call x86_thiscallcc void @thiscall_with_inalloca(i8* null, %Foo* inalloca %b) ; CHECK-DAG: xorl %ecx, %ecx ; CHECK: calll _thiscall_with_inalloca diff --git a/test/CodeGen/X86/inconsistent_landingpad.ll b/test/CodeGen/X86/inconsistent_landingpad.ll new file mode 100644 index 000000000000..495e999c4a95 --- /dev/null +++ b/test/CodeGen/X86/inconsistent_landingpad.ll @@ -0,0 +1,30 @@ +; RUN: not llvm-as -disable-output <%s 2>&1 | FileCheck %s + +define void @test() personality i32 (...)* @dummy_personality { +; CHECK: The landingpad instruction should have a consistent result type inside a function +entry: + invoke void @dummy1() + to label %next unwind label %unwind1 + +unwind1: + %lp1 = landingpad token + cleanup + br label %return + +next: + invoke void @dummy2() + to label %return unwind label %unwind2 + +unwind2: + %lp2 = landingpad { i8*, i32 } + cleanup + br label %return + +return: + ret void +} + +declare void @dummy1() +declare void @dummy2() + +declare i32 @dummy_personality(...) diff --git a/test/CodeGen/X86/inline-asm-2addr.ll b/test/CodeGen/X86/inline-asm-2addr.ll index 4a2c7fc5ebac..079f883186fb 100644 --- a/test/CodeGen/X86/inline-asm-2addr.ll +++ b/test/CodeGen/X86/inline-asm-2addr.ll @@ -1,9 +1,18 @@ -; RUN: llc < %s -march=x86-64 | not grep movq +; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s define i64 @t(i64 %a, i64 %b) nounwind ssp { entry: +; CHECK-LABEL: t: %asmtmp = tail call i64 asm "rorq $1,$0", "=r,J,0,~{dirflag},~{fpsr},~{flags},~{cc}"(i32 1, i64 %a) nounwind ; [#uses=1] +; CHECK: #APP +; CHECK-NEXT: rorq %[[REG1:.*]] +; CHECK-NEXT: #NO_APP %asmtmp1 = tail call i64 asm "rorq $1,$0", "=r,J,0,~{dirflag},~{fpsr},~{flags},~{cc}"(i32 1, i64 %b) nounwind ; [#uses=1] +; CHECK-NEXT: #APP +; CHECK-NEXT: rorq %[[REG2:.*]] +; CHECK-NEXT: #NO_APP %0 = add i64 %asmtmp1, %asmtmp ; [#uses=1] +; CHECK-NEXT: leaq (%[[REG2]],%[[REG1]]), %rax ret i64 %0 +; CHECK: retq } diff --git a/test/CodeGen/X86/inline-asm-sp-clobber-memcpy.ll b/test/CodeGen/X86/inline-asm-sp-clobber-memcpy.ll index b55571bcba09..970b9943948f 100644 --- a/test/CodeGen/X86/inline-asm-sp-clobber-memcpy.ll +++ b/test/CodeGen/X86/inline-asm-sp-clobber-memcpy.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -force-align-stack -mtriple i386-apple-darwin -mcpu=i486 | FileCheck %s +; RUN: llc < %s -stackrealign -mtriple i386-apple-darwin -mcpu=i486 | FileCheck %s %struct.foo = type { [88 x i8] } diff --git a/test/CodeGen/X86/inline-sse.ll b/test/CodeGen/X86/inline-sse.ll new file mode 100644 index 000000000000..78d6b762b5e5 --- /dev/null +++ b/test/CodeGen/X86/inline-sse.ll @@ -0,0 +1,34 @@ +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse | FileCheck %s --check-prefix=X32 +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X32 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-sse2 | FileCheck %s --check-prefix=X64 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X64 + +; PR16133 - we must treat XMM registers as v4f32 as SSE1 targets don't permit other vector types. + +define void @nop() nounwind { +; X32-LABEL: nop: +; X32: # BB#0: +; X32-NEXT: pushl %ebp +; X32-NEXT: movl %esp, %ebp +; X32-NEXT: andl $-16, %esp +; X32-NEXT: subl $32, %esp +; X32-NEXT: #APP +; X32-NEXT: #NO_APP +; X32-NEXT: movaps %xmm0, (%esp) +; X32-NEXT: movl %ebp, %esp +; X32-NEXT: popl %ebp +; X32-NEXT: retl +; +; X64-LABEL: nop: +; X64: # BB#0: +; X64-NEXT: subq $24, %rsp +; X64-NEXT: #APP +; X64-NEXT: #NO_APP +; X64-NEXT: movaps %xmm0, (%rsp) +; X64-NEXT: addq $24, %rsp +; X64-NEXT: retq + %1 = alloca <4 x float>, align 16 + %2 = call <4 x float> asm "", "=x,~{dirflag},~{fpsr},~{flags}"() + store <4 x float> %2, <4 x float>* %1, align 16 + ret void +} diff --git a/test/CodeGen/X86/insertps-from-constantpool.ll b/test/CodeGen/X86/insertps-from-constantpool.ll new file mode 100644 index 000000000000..cfcfeacad067 --- /dev/null +++ b/test/CodeGen/X86/insertps-from-constantpool.ll @@ -0,0 +1,20 @@ +; RUN: llc -mtriple=i686-unknown-unknown -mattr=+sse4.1 < %s | FileCheck %s --check-prefix=X32 +; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 < %s | FileCheck %s --check-prefix=X64 + +; Test for case where insertps folds the load of an insertion element from a constant pool. + +define <4 x float> @fold_from_constantpool(<4 x float> %a) { +; X32-LABEL: fold_from_constantpool: +; X32: # BB#0: +; X32-NEXT: insertps {{.*#+}} xmm0 = mem[0],xmm0[1,2,3] +; X32-NEXT: retl +; +; X64-LABEL: fold_from_constantpool: +; X64: # BB#0: +; X64-NEXT: insertps {{.*#+}} xmm0 = mem[0],xmm0[1,2,3] +; X64-NEXT: retq + %1 = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %a, <4 x float> , i8 64) + ret <4 x float> %1 +} + +declare <4 x float> @llvm.x86.sse41.insertps(<4 x float>, <4 x float>, i8) nounwind readnone diff --git a/test/CodeGen/X86/insertps-unfold-load-bug.ll b/test/CodeGen/X86/insertps-unfold-load-bug.ll new file mode 100644 index 000000000000..bf7c4bc4d7b9 --- /dev/null +++ b/test/CodeGen/X86/insertps-unfold-load-bug.ll @@ -0,0 +1,33 @@ +; RUN: llc -mtriple=i686-unknown-unknown -mattr=+sse4.1 < %s | FileCheck %s -check-prefix=X32 +; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 < %s | FileCheck %s -check-prefix=X64 + +; Test for case where insertps was folding the load of the insertion element, but a later optimization +; was then manipulating the load. + +define <4 x float> @insertps_unfold(<4 x float>* %v0, <4 x float>* %v1) { +; X32-LABEL: insertps_unfold: +; X32: # BB#0: +; X32-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X32-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero +; X32-NEXT: movaps (%eax), %xmm0 +; X32-NEXT: insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] +; X32-NEXT: addps %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: insertps_unfold: +; X64: # BB#0: +; X64-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero +; X64-NEXT: movaps (%rdi), %xmm0 +; X64-NEXT: insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] +; X64-NEXT: addps %xmm1, %xmm0 +; X64-NEXT: retq + %a = getelementptr inbounds <4 x float>, <4 x float>* %v1, i64 0, i64 1 + %b = load float, float* %a, align 4 + %c = insertelement <4 x float> undef, float %b, i32 0 + %d = load <4 x float>, <4 x float>* %v1, align 16 + %e = load <4 x float>, <4 x float>* %v0, align 16 + %f = shufflevector <4 x float> %e, <4 x float> %d, <4 x i32> + %g = fadd <4 x float> %c, %f + ret <4 x float> %g +} diff --git a/test/CodeGen/X86/int-intrinsic.ll b/test/CodeGen/X86/int-intrinsic.ll index 45a9b0f15c67..b253e6c5f3b0 100644 --- a/test/CodeGen/X86/int-intrinsic.ll +++ b/test/CodeGen/X86/int-intrinsic.ll @@ -11,7 +11,7 @@ bb.entry: ret void } -; CHECK: int $-128 +; CHECK: int $128 ; CHECK: ret define void @primitive_int128 () { bb.entry: diff --git a/test/CodeGen/X86/late-address-taken.ll b/test/CodeGen/X86/late-address-taken.ll new file mode 100644 index 000000000000..f98c53595abb --- /dev/null +++ b/test/CodeGen/X86/late-address-taken.ll @@ -0,0 +1,68 @@ +; RUN: llc -mtriple=x86_64-pc-windows-msvc < %s -enable-shrink-wrap=false | FileCheck %s +; Make sure shrink-wrapping does not break the lowering of exception handling. +; RUN: llc -mtriple=x86_64-pc-windows-msvc < %s -enable-shrink-wrap=true | FileCheck %s + +; Repro cases from PR25168 + +; test @catchret - catchret target is not address-taken until PEI +; splits it into lea/mov followed by ret. Make sure the MBB is +; handled, both by tempting BranchFolding to merge it with %early_out +; and delete it, and by checking that we emit a proper reference +; to it in the LEA + +declare void @ProcessCLRException() +declare void @f() + +define void @catchret(i1 %b) personality void ()* @ProcessCLRException { +entry: + br i1 %b, label %body, label %early_out +early_out: + ret void +body: + invoke void @f() + to label %exit unwind label %catch.pad +catch.pad: + %cs1 = catchswitch within none [label %catch.body] unwind to caller +catch.body: + %catch = catchpad within %cs1 [i32 33554467] + catchret from %catch to label %exit +exit: + ret void +} +; CHECK-LABEL: catchret: # @catchret +; CHECK: [[Exit:^[^ :]+]]: # Block address taken +; CHECK-NEXT: # %exit +; CHECK: # %catch.body +; CHECK: .seh_endprolog +; CHECK: leaq [[Exit]](%rip), %rax +; CHECK: retq # CATCHRET + + +; test @setjmp - similar to @catchret, but the MBB in question +; is the one generated when the setjmp's block is split + +@buf = internal global [5 x i8*] zeroinitializer +declare i8* @llvm.frameaddress(i32) nounwind readnone +declare i8* @llvm.stacksave() nounwind +declare i32 @llvm.eh.sjlj.setjmp(i8*) nounwind +declare void @llvm.eh.sjlj.longjmp(i8*) nounwind + +define void @setjmp(i1 %b) nounwind { +entry: + br i1 %b, label %early_out, label %sj +early_out: + ret void +sj: + %fp = call i8* @llvm.frameaddress(i32 0) + store i8* %fp, i8** getelementptr inbounds ([5 x i8*], [5 x i8*]* @buf, i64 0, i64 0), align 16 + %sp = call i8* @llvm.stacksave() + store i8* %sp, i8** getelementptr inbounds ([5 x i8*], [5 x i8*]* @buf, i64 0, i64 2), align 16 + call i32 @llvm.eh.sjlj.setjmp(i8* bitcast ([5 x i8*]* @buf to i8*)) + ret void +} +; CHECK-LABEL: setjmp: # @setjmp +; CHECK: # %sj +; CHECK: leaq [[Label:\..+]](%rip), %[[Reg:.+]]{{$}} +; CHECK-NEXT: movq %[[Reg]], buf +; CHECK: {{^}}[[Label]]: # Block address taken +; CHECK-NEXT: # %sj diff --git a/test/CodeGen/X86/lea-opt.ll b/test/CodeGen/X86/lea-opt.ll new file mode 100644 index 000000000000..571f2d9084c4 --- /dev/null +++ b/test/CodeGen/X86/lea-opt.ll @@ -0,0 +1,131 @@ +; RUN: llc < %s -mtriple=x86_64-linux -enable-x86-lea-opt | FileCheck %s + +%struct.anon1 = type { i32, i32, i32 } +%struct.anon2 = type { i32, [32 x i32], i32 } + +@arr1 = external global [65 x %struct.anon1], align 16 +@arr2 = external global [65 x %struct.anon2], align 16 + +define void @test1(i64 %x) nounwind { +entry: + %a = getelementptr inbounds [65 x %struct.anon1], [65 x %struct.anon1]* @arr1, i64 0, i64 %x, i32 0 + %tmp = load i32, i32* %a, align 4 + %b = getelementptr inbounds [65 x %struct.anon1], [65 x %struct.anon1]* @arr1, i64 0, i64 %x, i32 1 + %tmp1 = load i32, i32* %b, align 4 + %sub = sub i32 %tmp, %tmp1 + %c = getelementptr inbounds [65 x %struct.anon1], [65 x %struct.anon1]* @arr1, i64 0, i64 %x, i32 2 + %tmp2 = load i32, i32* %c, align 4 + %add = add nsw i32 %sub, %tmp2 + switch i32 %add, label %sw.epilog [ + i32 1, label %sw.bb.1 + i32 2, label %sw.bb.2 + ] + +sw.bb.1: ; preds = %entry + store i32 111, i32* %b, align 4 + store i32 222, i32* %c, align 4 + br label %sw.epilog + +sw.bb.2: ; preds = %entry + store i32 333, i32* %b, align 4 + store i32 444, i32* %c, align 4 + br label %sw.epilog + +sw.epilog: ; preds = %sw.bb.2, %sw.bb.1, %entry + ret void +; CHECK-LABEL: test1: +; CHECK: leaq (%rdi,%rdi,2), [[REG1:%[a-z]+]] +; CHECK: movl arr1(,[[REG1]],4), {{.*}} +; CHECK: leaq arr1+4(,[[REG1]],4), [[REG2:%[a-z]+]] +; CHECK: subl arr1+4(,[[REG1]],4), {{.*}} +; CHECK: leaq arr1+8(,[[REG1]],4), [[REG3:%[a-z]+]] +; CHECK: addl arr1+8(,[[REG1]],4), {{.*}} +; CHECK: movl ${{[1-4]+}}, ([[REG2]]) +; CHECK: movl ${{[1-4]+}}, ([[REG3]]) +; CHECK: movl ${{[1-4]+}}, ([[REG2]]) +; CHECK: movl ${{[1-4]+}}, ([[REG3]]) +} + +define void @test2(i64 %x) nounwind optsize { +entry: + %a = getelementptr inbounds [65 x %struct.anon1], [65 x %struct.anon1]* @arr1, i64 0, i64 %x, i32 0 + %tmp = load i32, i32* %a, align 4 + %b = getelementptr inbounds [65 x %struct.anon1], [65 x %struct.anon1]* @arr1, i64 0, i64 %x, i32 1 + %tmp1 = load i32, i32* %b, align 4 + %sub = sub i32 %tmp, %tmp1 + %c = getelementptr inbounds [65 x %struct.anon1], [65 x %struct.anon1]* @arr1, i64 0, i64 %x, i32 2 + %tmp2 = load i32, i32* %c, align 4 + %add = add nsw i32 %sub, %tmp2 + switch i32 %add, label %sw.epilog [ + i32 1, label %sw.bb.1 + i32 2, label %sw.bb.2 + ] + +sw.bb.1: ; preds = %entry + store i32 111, i32* %b, align 4 + store i32 222, i32* %c, align 4 + br label %sw.epilog + +sw.bb.2: ; preds = %entry + store i32 333, i32* %b, align 4 + store i32 444, i32* %c, align 4 + br label %sw.epilog + +sw.epilog: ; preds = %sw.bb.2, %sw.bb.1, %entry + ret void +; CHECK-LABEL: test2: +; CHECK: leaq (%rdi,%rdi,2), [[REG1:%[a-z]+]] +; CHECK: leaq arr1+4(,[[REG1]],4), [[REG2:%[a-z]+]] +; CHECK: movl -4([[REG2]]), {{.*}} +; CHECK: subl ([[REG2]]), {{.*}} +; CHECK: leaq arr1+8(,[[REG1]],4), [[REG3:%[a-z]+]] +; CHECK: addl ([[REG3]]), {{.*}} +; CHECK: movl ${{[1-4]+}}, ([[REG2]]) +; CHECK: movl ${{[1-4]+}}, ([[REG3]]) +; CHECK: movl ${{[1-4]+}}, ([[REG2]]) +; CHECK: movl ${{[1-4]+}}, ([[REG3]]) +} + +; Check that LEA optimization pass takes into account a resultant address +; displacement when choosing a LEA instruction for replacing a redundant +; address recalculation. + +define void @test3(i64 %x) nounwind optsize { +entry: + %a = getelementptr inbounds [65 x %struct.anon2], [65 x %struct.anon2]* @arr2, i64 0, i64 %x, i32 2 + %tmp = load i32, i32* %a, align 4 + %b = getelementptr inbounds [65 x %struct.anon2], [65 x %struct.anon2]* @arr2, i64 0, i64 %x, i32 0 + %tmp1 = load i32, i32* %b, align 4 + %add = add nsw i32 %tmp, %tmp1 + switch i32 %add, label %sw.epilog [ + i32 1, label %sw.bb.1 + i32 2, label %sw.bb.2 + ] + +sw.bb.1: ; preds = %entry + store i32 111, i32* %a, align 4 + store i32 222, i32* %b, align 4 + br label %sw.epilog + +sw.bb.2: ; preds = %entry + store i32 333, i32* %a, align 4 + store i32 444, i32* %b, align 4 + br label %sw.epilog + +sw.epilog: ; preds = %sw.bb.2, %sw.bb.1, %entry + ret void +; CHECK-LABEL: test3: +; CHECK: imulq {{.*}}, [[REG1:%[a-z]+]] +; CHECK: leaq arr2+132([[REG1]]), [[REG2:%[a-z]+]] +; CHECK: leaq arr2([[REG1]]), [[REG3:%[a-z]+]] + +; REG3's definition is closer to movl than REG2's, but the pass still chooses +; REG2 because it provides the resultant address displacement fitting 1 byte. + +; CHECK: movl ([[REG2]]), {{.*}} +; CHECK: addl ([[REG3]]), {{.*}} +; CHECK: movl ${{[1-4]+}}, ([[REG2]]) +; CHECK: movl ${{[1-4]+}}, ([[REG3]]) +; CHECK: movl ${{[1-4]+}}, ([[REG2]]) +; CHECK: movl ${{[1-4]+}}, ([[REG3]]) +} diff --git a/test/CodeGen/X86/lit.local.cfg b/test/CodeGen/X86/lit.local.cfg index 8ed58f119c4f..4a1dd86abc45 100644 --- a/test/CodeGen/X86/lit.local.cfg +++ b/test/CodeGen/X86/lit.local.cfg @@ -4,7 +4,7 @@ # # It should be possible to remove this override once all the bots have cycled # cleanly. -config.suffixes = ['.ll', '.test', '.txt'] +config.suffixes = ['.ll', '.mir', '.test', '.txt'] if not 'X86' in config.root.targets: config.unsupported = True diff --git a/test/CodeGen/X86/localescape.ll b/test/CodeGen/X86/localescape.ll new file mode 100644 index 000000000000..3cd174df0b71 --- /dev/null +++ b/test/CodeGen/X86/localescape.ll @@ -0,0 +1,143 @@ +; RUN: llc -mtriple=x86_64-windows-msvc < %s | FileCheck %s --check-prefix=X64 +; RUN: llc -mtriple=i686-windows-msvc < %s | FileCheck %s --check-prefix=X86 + +declare i8* @llvm.frameaddress(i32) +declare void @llvm.localescape(...) +declare i8* @llvm.localaddress() +declare i8* @llvm.localrecover(i8*, i8*, i32) +declare i32 @printf(i8*, ...) + +@str = internal constant [10 x i8] c"asdf: %d\0A\00" + +define void @print_framealloc_from_fp(i8* %fp) { + %a.i8 = call i8* @llvm.localrecover(i8* bitcast (void(i32)* @alloc_func to i8*), i8* %fp, i32 0) + %a = bitcast i8* %a.i8 to i32* + %a.val = load i32, i32* %a + call i32 (i8*, ...) @printf(i8* getelementptr ([10 x i8], [10 x i8]* @str, i32 0, i32 0), i32 %a.val) + %b.i8 = call i8* @llvm.localrecover(i8* bitcast (void(i32)* @alloc_func to i8*), i8* %fp, i32 1) + %b = bitcast i8* %b.i8 to i32* + %b.val = load i32, i32* %b + call i32 (i8*, ...) @printf(i8* getelementptr ([10 x i8], [10 x i8]* @str, i32 0, i32 0), i32 %b.val) + store i32 42, i32* %b + %b2 = getelementptr i32, i32* %b, i32 1 + %b2.val = load i32, i32* %b2 + call i32 (i8*, ...) @printf(i8* getelementptr ([10 x i8], [10 x i8]* @str, i32 0, i32 0), i32 %b2.val) + ret void +} + +; X64-LABEL: print_framealloc_from_fp: +; X64: movq %rcx, %[[parent_fp:[a-z]+]] +; X64: movl .Lalloc_func$frame_escape_0(%[[parent_fp]]), %edx +; X64: leaq {{.*}}(%rip), %[[str:[a-z]+]] +; X64: movq %[[str]], %rcx +; X64: callq printf +; X64: movl .Lalloc_func$frame_escape_1(%[[parent_fp]]), %edx +; X64: movq %[[str]], %rcx +; X64: callq printf +; X64: movl $42, .Lalloc_func$frame_escape_1(%[[parent_fp]]) +; X64: retq + +; X86-LABEL: print_framealloc_from_fp: +; X86: pushl %esi +; X86: subl $8, %esp +; X86: movl 16(%esp), %esi +; X86: movl Lalloc_func$frame_escape_0(%esi), %eax +; X86: movl %eax, 4(%esp) +; X86: movl $_str, (%esp) +; X86: calll _printf +; X86: movl Lalloc_func$frame_escape_1(%esi), %eax +; X86: movl %eax, 4(%esp) +; X86: movl $_str, (%esp) +; X86: calll _printf +; X86: movl $42, Lalloc_func$frame_escape_1(%esi) +; X86: movl $4, %eax +; X86: movl Lalloc_func$frame_escape_1(%esi,%eax), %eax +; X86: movl %eax, 4(%esp) +; X86: movl $_str, (%esp) +; X86: calll _printf +; X86: addl $8, %esp +; X86: popl %esi +; X86: retl + +define void @alloc_func(i32 %n) { + %a = alloca i32 + %b = alloca i32, i32 2 + call void (...) @llvm.localescape(i32* %a, i32* %b) + store i32 42, i32* %a + store i32 13, i32* %b + + ; Force usage of EBP with a dynamic alloca. + alloca i8, i32 %n + + %lp = call i8* @llvm.localaddress() + call void @print_framealloc_from_fp(i8* %lp) + ret void +} + +; X64-LABEL: alloc_func: +; X64: pushq %rbp +; X64: subq $16, %rsp +; X64: .seh_stackalloc 16 +; X64: leaq 16(%rsp), %rbp +; X64: .seh_setframe 5, 16 +; X64: .Lalloc_func$frame_escape_0 = -4 +; X64: .Lalloc_func$frame_escape_1 = -12 +; X64: movl $42, -4(%rbp) +; X64: movl $13, -12(%rbp) +; X64: movq %rbp, %rcx +; X64: callq print_framealloc_from_fp +; X64: retq + +; X86-LABEL: alloc_func: +; X86: pushl %ebp +; X86: movl %esp, %ebp +; X86: subl $12, %esp +; X86: Lalloc_func$frame_escape_0 = -4 +; X86: Lalloc_func$frame_escape_1 = -12 +; X86: movl $42, -4(%ebp) +; X86: movl $13, -12(%ebp) +; X86: pushl %ebp +; X86: calll _print_framealloc_from_fp +; X86: movl %ebp, %esp +; X86: popl %ebp +; X86: retl + +; Helper to make this a complete program so it can be compiled and tested. +define i32 @main() { + call void @alloc_func(i32 3) + ret i32 0 +} + +define void @alloc_func_no_frameaddr() { + %a = alloca i32 + %b = alloca i32 + call void (...) @llvm.localescape(i32* %a, i32* %b) + store i32 42, i32* %a + store i32 13, i32* %b + call void @print_framealloc_from_fp(i8* null) + ret void +} + +; X64-LABEL: alloc_func_no_frameaddr: +; X64: subq $40, %rsp +; X64: .seh_stackalloc 40 +; X64: .seh_endprologue +; X64: .Lalloc_func_no_frameaddr$frame_escape_0 = 36 +; X64: .Lalloc_func_no_frameaddr$frame_escape_1 = 32 +; X64: movl $42, 36(%rsp) +; X64: movl $13, 32(%rsp) +; X64: xorl %ecx, %ecx +; X64: callq print_framealloc_from_fp +; X64: addq $40, %rsp +; X64: retq + +; X86-LABEL: alloc_func_no_frameaddr: +; X86: subl $12, %esp +; X86: Lalloc_func_no_frameaddr$frame_escape_0 = 8 +; X86: Lalloc_func_no_frameaddr$frame_escape_1 = 4 +; X86: movl $42, 8(%esp) +; X86: movl $13, 4(%esp) +; X86: movl $0, (%esp) +; X86: calll _print_framealloc_from_fp +; X86: addl $12, %esp +; X86: retl diff --git a/test/CodeGen/X86/lower-vec-shift-2.ll b/test/CodeGen/X86/lower-vec-shift-2.ll index fb8fbba71fca..281461577004 100644 --- a/test/CodeGen/X86/lower-vec-shift-2.ll +++ b/test/CodeGen/X86/lower-vec-shift-2.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+sse2 < %s | FileCheck %s --check-prefix=SSE2 ; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+avx < %s | FileCheck %s --check-prefix=AVX diff --git a/test/CodeGen/X86/lsr-static-addr.ll b/test/CodeGen/X86/lsr-static-addr.ll index 1765ed7871d8..97451e5573fe 100644 --- a/test/CodeGen/X86/lsr-static-addr.ll +++ b/test/CodeGen/X86/lsr-static-addr.ll @@ -18,7 +18,7 @@ ; ATOM-NEXT: movsd A(,%rax,8) ; ATOM-NEXT: mulsd ; ATOM-NEXT: movsd -; ATOM-NEXT: leaq 1(%rax), %rax +; ATOM-NEXT: incq %rax @A = external global [0 x double] diff --git a/test/CodeGen/X86/machine-combiner-int-vec.ll b/test/CodeGen/X86/machine-combiner-int-vec.ll new file mode 100644 index 000000000000..dc1ce77e13b7 --- /dev/null +++ b/test/CodeGen/X86/machine-combiner-int-vec.ll @@ -0,0 +1,112 @@ +; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=sse2 < %s | FileCheck %s --check-prefix=SSE +; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=avx2 < %s | FileCheck %s --check-prefix=AVX + +; Verify that 128-bit vector logical ops are reassociated. + +define <4 x i32> @reassociate_and_v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, <4 x i32> %x3) { +; SSE-LABEL: reassociate_and_v4i32: +; SSE: # BB#0: +; SSE-NEXT: paddd %xmm1, %xmm0 +; SSE-NEXT: pand %xmm3, %xmm2 +; SSE-NEXT: pand %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: reassociate_and_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpand %xmm3, %xmm2, %xmm1 +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + + %t0 = add <4 x i32> %x0, %x1 + %t1 = and <4 x i32> %x2, %t0 + %t2 = and <4 x i32> %x3, %t1 + ret <4 x i32> %t2 +} + +define <4 x i32> @reassociate_or_v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, <4 x i32> %x3) { +; SSE-LABEL: reassociate_or_v4i32: +; SSE: # BB#0: +; SSE-NEXT: paddd %xmm1, %xmm0 +; SSE-NEXT: por %xmm3, %xmm2 +; SSE-NEXT: por %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: reassociate_or_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpor %xmm3, %xmm2, %xmm1 +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + + %t0 = add <4 x i32> %x0, %x1 + %t1 = or <4 x i32> %x2, %t0 + %t2 = or <4 x i32> %x3, %t1 + ret <4 x i32> %t2 +} + +define <4 x i32> @reassociate_xor_v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, <4 x i32> %x3) { +; SSE-LABEL: reassociate_xor_v4i32: +; SSE: # BB#0: +; SSE-NEXT: paddd %xmm1, %xmm0 +; SSE-NEXT: pxor %xmm3, %xmm2 +; SSE-NEXT: pxor %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: reassociate_xor_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpxor %xmm3, %xmm2, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + + %t0 = add <4 x i32> %x0, %x1 + %t1 = xor <4 x i32> %x2, %t0 + %t2 = xor <4 x i32> %x3, %t1 + ret <4 x i32> %t2 +} + +; Verify that 256-bit vector logical ops are reassociated. + +define <8 x i32> @reassociate_and_v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, <8 x i32> %x3) { +; AVX-LABEL: reassociate_and_v8i32: +; AVX: # BB#0: +; AVX-NEXT: vpaddd %ymm1, %ymm0, %ymm0 +; AVX-NEXT: vpand %ymm3, %ymm2, %ymm1 +; AVX-NEXT: vpand %ymm1, %ymm0, %ymm0 +; AVX-NEXT: retq + + %t0 = add <8 x i32> %x0, %x1 + %t1 = and <8 x i32> %x2, %t0 + %t2 = and <8 x i32> %x3, %t1 + ret <8 x i32> %t2 +} + +define <8 x i32> @reassociate_or_v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, <8 x i32> %x3) { +; AVX-LABEL: reassociate_or_v8i32: +; AVX: # BB#0: +; AVX-NEXT: vpaddd %ymm1, %ymm0, %ymm0 +; AVX-NEXT: vpor %ymm3, %ymm2, %ymm1 +; AVX-NEXT: vpor %ymm1, %ymm0, %ymm0 +; AVX-NEXT: retq + + %t0 = add <8 x i32> %x0, %x1 + %t1 = or <8 x i32> %x2, %t0 + %t2 = or <8 x i32> %x3, %t1 + ret <8 x i32> %t2 +} + +define <8 x i32> @reassociate_xor_v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, <8 x i32> %x3) { +; AVX-LABEL: reassociate_xor_v8i32: +; AVX: # BB#0: +; AVX-NEXT: vpaddd %ymm1, %ymm0, %ymm0 +; AVX-NEXT: vpxor %ymm3, %ymm2, %ymm1 +; AVX-NEXT: vpxor %ymm1, %ymm0, %ymm0 +; AVX-NEXT: retq + + %t0 = add <8 x i32> %x0, %x1 + %t1 = xor <8 x i32> %x2, %t0 + %t2 = xor <8 x i32> %x3, %t1 + ret <8 x i32> %t2 +} + diff --git a/test/CodeGen/X86/machine-combiner-int.ll b/test/CodeGen/X86/machine-combiner-int.ll new file mode 100644 index 000000000000..4a1ba1a980ae --- /dev/null +++ b/test/CodeGen/X86/machine-combiner-int.ll @@ -0,0 +1,194 @@ +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -stop-after machine-combiner -o /dev/null 2>&1 | FileCheck %s --check-prefix=DEAD + +; Verify that integer multiplies are reassociated. The first multiply in +; each test should be independent of the result of the preceding add (lea). + +; TODO: This test does not actually test i16 machine instruction reassociation +; because the operands are being promoted to i32 types. + +define i16 @reassociate_muls_i16(i16 %x0, i16 %x1, i16 %x2, i16 %x3) { +; CHECK-LABEL: reassociate_muls_i16: +; CHECK: # BB#0: +; CHECK-NEXT: leal (%rdi,%rsi), %eax +; CHECK-NEXT: imull %ecx, %edx +; CHECK-NEXT: imull %edx, %eax +; CHECK-NEXT: retq + %t0 = add i16 %x0, %x1 + %t1 = mul i16 %x2, %t0 + %t2 = mul i16 %x3, %t1 + ret i16 %t2 +} + +define i32 @reassociate_muls_i32(i32 %x0, i32 %x1, i32 %x2, i32 %x3) { +; CHECK-LABEL: reassociate_muls_i32: +; CHECK: # BB#0: +; CHECK-NEXT: leal (%rdi,%rsi), %eax +; CHECK-NEXT: imull %ecx, %edx +; CHECK-NEXT: imull %edx, %eax +; CHECK-NEXT: retq + +; DEAD: ADD32rr +; DEAD-NEXT: IMUL32rr{{.*}}implicit-def dead %eflags +; DEAD-NEXT: IMUL32rr{{.*}}implicit-def dead %eflags + + %t0 = add i32 %x0, %x1 + %t1 = mul i32 %x2, %t0 + %t2 = mul i32 %x3, %t1 + ret i32 %t2 +} + +define i64 @reassociate_muls_i64(i64 %x0, i64 %x1, i64 %x2, i64 %x3) { +; CHECK-LABEL: reassociate_muls_i64: +; CHECK: # BB#0: +; CHECK-NEXT: leaq (%rdi,%rsi), %rax +; CHECK-NEXT: imulq %rcx, %rdx +; CHECK-NEXT: imulq %rdx, %rax +; CHECK-NEXT: retq + %t0 = add i64 %x0, %x1 + %t1 = mul i64 %x2, %t0 + %t2 = mul i64 %x3, %t1 + ret i64 %t2 +} + +; Verify that integer 'ands' are reassociated. The first 'and' in +; each test should be independent of the result of the preceding sub. + +define i8 @reassociate_ands_i8(i8 %x0, i8 %x1, i8 %x2, i8 %x3) { +; CHECK-LABEL: reassociate_ands_i8: +; CHECK: # BB#0: +; CHECK-NEXT: subb %sil, %dil +; CHECK-NEXT: andb %cl, %dl +; CHECK-NEXT: andb %dil, %dl +; CHECK_NEXT: movb %dx, %ax +; CHECK_NEXT: retq + %t0 = sub i8 %x0, %x1 + %t1 = and i8 %x2, %t0 + %t2 = and i8 %x3, %t1 + ret i8 %t2 +} + +; TODO: No way to test i16? These appear to always get promoted to i32. + +define i32 @reassociate_ands_i32(i32 %x0, i32 %x1, i32 %x2, i32 %x3) { +; CHECK-LABEL: reassociate_ands_i32: +; CHECK: # BB#0: +; CHECK-NEXT: subl %esi, %edi +; CHECK-NEXT: andl %ecx, %edx +; CHECK-NEXT: andl %edi, %edx +; CHECK_NEXT: movl %edx, %eax +; CHECK_NEXT: retq + %t0 = sub i32 %x0, %x1 + %t1 = and i32 %x2, %t0 + %t2 = and i32 %x3, %t1 + ret i32 %t2 +} + +define i64 @reassociate_ands_i64(i64 %x0, i64 %x1, i64 %x2, i64 %x3) { +; CHECK-LABEL: reassociate_ands_i64: +; CHECK: # BB#0: +; CHECK-NEXT: subq %rsi, %rdi +; CHECK-NEXT: andq %rcx, %rdx +; CHECK-NEXT: andq %rdi, %rdx +; CHECK-NEXT: movq %rdx, %rax +; CHECK_NEXT: retq + %t0 = sub i64 %x0, %x1 + %t1 = and i64 %x2, %t0 + %t2 = and i64 %x3, %t1 + ret i64 %t2 +} + +; Verify that integer 'ors' are reassociated. The first 'or' in +; each test should be independent of the result of the preceding sub. + +define i8 @reassociate_ors_i8(i8 %x0, i8 %x1, i8 %x2, i8 %x3) { +; CHECK-LABEL: reassociate_ors_i8: +; CHECK: # BB#0: +; CHECK-NEXT: subb %sil, %dil +; CHECK-NEXT: orb %cl, %dl +; CHECK-NEXT: orb %dil, %dl +; CHECK_NEXT: movb %dx, %ax +; CHECK_NEXT: retq + %t0 = sub i8 %x0, %x1 + %t1 = or i8 %x2, %t0 + %t2 = or i8 %x3, %t1 + ret i8 %t2 +} + +; TODO: No way to test i16? These appear to always get promoted to i32. + +define i32 @reassociate_ors_i32(i32 %x0, i32 %x1, i32 %x2, i32 %x3) { +; CHECK-LABEL: reassociate_ors_i32: +; CHECK: # BB#0: +; CHECK-NEXT: subl %esi, %edi +; CHECK-NEXT: orl %ecx, %edx +; CHECK-NEXT: orl %edi, %edx +; CHECK_NEXT: movl %edx, %eax +; CHECK_NEXT: retq + %t0 = sub i32 %x0, %x1 + %t1 = or i32 %x2, %t0 + %t2 = or i32 %x3, %t1 + ret i32 %t2 +} + +define i64 @reassociate_ors_i64(i64 %x0, i64 %x1, i64 %x2, i64 %x3) { +; CHECK-LABEL: reassociate_ors_i64: +; CHECK: # BB#0: +; CHECK-NEXT: subq %rsi, %rdi +; CHECK-NEXT: orq %rcx, %rdx +; CHECK-NEXT: orq %rdi, %rdx +; CHECK-NEXT: movq %rdx, %rax +; CHECK_NEXT: retq + %t0 = sub i64 %x0, %x1 + %t1 = or i64 %x2, %t0 + %t2 = or i64 %x3, %t1 + ret i64 %t2 +} + +; Verify that integer 'xors' are reassociated. The first 'xor' in +; each test should be independent of the result of the preceding sub. + +define i8 @reassociate_xors_i8(i8 %x0, i8 %x1, i8 %x2, i8 %x3) { +; CHECK-LABEL: reassociate_xors_i8: +; CHECK: # BB#0: +; CHECK-NEXT: subb %sil, %dil +; CHECK-NEXT: xorb %cl, %dl +; CHECK-NEXT: xorb %dil, %dl +; CHECK_NEXT: movb %dx, %ax +; CHECK_NEXT: retq + %t0 = sub i8 %x0, %x1 + %t1 = xor i8 %x2, %t0 + %t2 = xor i8 %x3, %t1 + ret i8 %t2 +} + +; TODO: No way to test i16? These appear to always get promoted to i32. + +define i32 @reassociate_xors_i32(i32 %x0, i32 %x1, i32 %x2, i32 %x3) { +; CHECK-LABEL: reassociate_xors_i32: +; CHECK: # BB#0: +; CHECK-NEXT: subl %esi, %edi +; CHECK-NEXT: xorl %ecx, %edx +; CHECK-NEXT: xorl %edi, %edx +; CHECK_NEXT: movl %edx, %eax +; CHECK_NEXT: retq + %t0 = sub i32 %x0, %x1 + %t1 = xor i32 %x2, %t0 + %t2 = xor i32 %x3, %t1 + ret i32 %t2 +} + +define i64 @reassociate_xors_i64(i64 %x0, i64 %x1, i64 %x2, i64 %x3) { +; CHECK-LABEL: reassociate_xors_i64: +; CHECK: # BB#0: +; CHECK-NEXT: subq %rsi, %rdi +; CHECK-NEXT: xorq %rcx, %rdx +; CHECK-NEXT: xorq %rdi, %rdx +; CHECK-NEXT: movq %rdx, %rax +; CHECK_NEXT: retq + %t0 = sub i64 %x0, %x1 + %t1 = xor i64 %x2, %t0 + %t2 = xor i64 %x3, %t1 + ret i64 %t2 +} + diff --git a/test/CodeGen/X86/machine-combiner.ll b/test/CodeGen/X86/machine-combiner.ll index 0943bebbb099..3fbb233696c8 100644 --- a/test/CodeGen/X86/machine-combiner.ll +++ b/test/CodeGen/X86/machine-combiner.ll @@ -144,7 +144,7 @@ define float @reassociate_adds6(float %x0, float %x1, float %x2, float %x3) { ret float %t2 } -; Verify that SSE and AVX scalar single-precison multiplies are reassociated. +; Verify that SSE and AVX scalar single-precision multiplies are reassociated. define float @reassociate_muls1(float %x0, float %x1, float %x2, float %x3) { ; SSE-LABEL: reassociate_muls1: @@ -166,7 +166,7 @@ define float @reassociate_muls1(float %x0, float %x1, float %x2, float %x3) { ret float %t2 } -; Verify that SSE and AVX scalar double-precison adds are reassociated. +; Verify that SSE and AVX scalar double-precision adds are reassociated. define double @reassociate_adds_double(double %x0, double %x1, double %x2, double %x3) { ; SSE-LABEL: reassociate_adds_double: @@ -188,7 +188,7 @@ define double @reassociate_adds_double(double %x0, double %x1, double %x2, doubl ret double %t2 } -; Verify that SSE and AVX scalar double-precison multiplies are reassociated. +; Verify that SSE and AVX scalar double-precision multiplies are reassociated. define double @reassociate_muls_double(double %x0, double %x1, double %x2, double %x3) { ; SSE-LABEL: reassociate_muls_double: @@ -210,3 +210,464 @@ define double @reassociate_muls_double(double %x0, double %x1, double %x2, doubl ret double %t2 } +; Verify that SSE and AVX 128-bit vector single-precision adds are reassociated. + +define <4 x float> @reassociate_adds_v4f32(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) { +; SSE-LABEL: reassociate_adds_v4f32: +; SSE: # BB#0: +; SSE-NEXT: mulps %xmm1, %xmm0 +; SSE-NEXT: addps %xmm3, %xmm2 +; SSE-NEXT: addps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: reassociate_adds_v4f32: +; AVX: # BB#0: +; AVX-NEXT: vmulps %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vaddps %xmm3, %xmm2, %xmm1 +; AVX-NEXT: vaddps %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %t0 = fmul <4 x float> %x0, %x1 + %t1 = fadd <4 x float> %x2, %t0 + %t2 = fadd <4 x float> %x3, %t1 + ret <4 x float> %t2 +} + +; Verify that SSE and AVX 128-bit vector double-precision adds are reassociated. + +define <2 x double> @reassociate_adds_v2f64(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, <2 x double> %x3) { +; SSE-LABEL: reassociate_adds_v2f64: +; SSE: # BB#0: +; SSE-NEXT: mulpd %xmm1, %xmm0 +; SSE-NEXT: addpd %xmm3, %xmm2 +; SSE-NEXT: addpd %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: reassociate_adds_v2f64: +; AVX: # BB#0: +; AVX-NEXT: vmulpd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vaddpd %xmm3, %xmm2, %xmm1 +; AVX-NEXT: vaddpd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %t0 = fmul <2 x double> %x0, %x1 + %t1 = fadd <2 x double> %x2, %t0 + %t2 = fadd <2 x double> %x3, %t1 + ret <2 x double> %t2 +} + +; Verify that SSE and AVX 128-bit vector single-precision multiplies are reassociated. + +define <4 x float> @reassociate_muls_v4f32(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) { +; SSE-LABEL: reassociate_muls_v4f32: +; SSE: # BB#0: +; SSE-NEXT: addps %xmm1, %xmm0 +; SSE-NEXT: mulps %xmm3, %xmm2 +; SSE-NEXT: mulps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: reassociate_muls_v4f32: +; AVX: # BB#0: +; AVX-NEXT: vaddps %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmulps %xmm3, %xmm2, %xmm1 +; AVX-NEXT: vmulps %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %t0 = fadd <4 x float> %x0, %x1 + %t1 = fmul <4 x float> %x2, %t0 + %t2 = fmul <4 x float> %x3, %t1 + ret <4 x float> %t2 +} + +; Verify that SSE and AVX 128-bit vector double-precision multiplies are reassociated. + +define <2 x double> @reassociate_muls_v2f64(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, <2 x double> %x3) { +; SSE-LABEL: reassociate_muls_v2f64: +; SSE: # BB#0: +; SSE-NEXT: addpd %xmm1, %xmm0 +; SSE-NEXT: mulpd %xmm3, %xmm2 +; SSE-NEXT: mulpd %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: reassociate_muls_v2f64: +; AVX: # BB#0: +; AVX-NEXT: vaddpd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmulpd %xmm3, %xmm2, %xmm1 +; AVX-NEXT: vmulpd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %t0 = fadd <2 x double> %x0, %x1 + %t1 = fmul <2 x double> %x2, %t0 + %t2 = fmul <2 x double> %x3, %t1 + ret <2 x double> %t2 +} + +; Verify that AVX 256-bit vector single-precision adds are reassociated. + +define <8 x float> @reassociate_adds_v8f32(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, <8 x float> %x3) { +; AVX-LABEL: reassociate_adds_v8f32: +; AVX: # BB#0: +; AVX-NEXT: vmulps %ymm1, %ymm0, %ymm0 +; AVX-NEXT: vaddps %ymm3, %ymm2, %ymm1 +; AVX-NEXT: vaddps %ymm1, %ymm0, %ymm0 +; AVX-NEXT: retq + %t0 = fmul <8 x float> %x0, %x1 + %t1 = fadd <8 x float> %x2, %t0 + %t2 = fadd <8 x float> %x3, %t1 + ret <8 x float> %t2 +} + +; Verify that AVX 256-bit vector double-precision adds are reassociated. + +define <4 x double> @reassociate_adds_v4f64(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, <4 x double> %x3) { +; AVX-LABEL: reassociate_adds_v4f64: +; AVX: # BB#0: +; AVX-NEXT: vmulpd %ymm1, %ymm0, %ymm0 +; AVX-NEXT: vaddpd %ymm3, %ymm2, %ymm1 +; AVX-NEXT: vaddpd %ymm1, %ymm0, %ymm0 +; AVX-NEXT: retq + %t0 = fmul <4 x double> %x0, %x1 + %t1 = fadd <4 x double> %x2, %t0 + %t2 = fadd <4 x double> %x3, %t1 + ret <4 x double> %t2 +} + +; Verify that AVX 256-bit vector single-precision multiplies are reassociated. + +define <8 x float> @reassociate_muls_v8f32(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, <8 x float> %x3) { +; AVX-LABEL: reassociate_muls_v8f32: +; AVX: # BB#0: +; AVX-NEXT: vaddps %ymm1, %ymm0, %ymm0 +; AVX-NEXT: vmulps %ymm3, %ymm2, %ymm1 +; AVX-NEXT: vmulps %ymm1, %ymm0, %ymm0 +; AVX-NEXT: retq + %t0 = fadd <8 x float> %x0, %x1 + %t1 = fmul <8 x float> %x2, %t0 + %t2 = fmul <8 x float> %x3, %t1 + ret <8 x float> %t2 +} + +; Verify that AVX 256-bit vector double-precision multiplies are reassociated. + +define <4 x double> @reassociate_muls_v4f64(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, <4 x double> %x3) { +; AVX-LABEL: reassociate_muls_v4f64: +; AVX: # BB#0: +; AVX-NEXT: vaddpd %ymm1, %ymm0, %ymm0 +; AVX-NEXT: vmulpd %ymm3, %ymm2, %ymm1 +; AVX-NEXT: vmulpd %ymm1, %ymm0, %ymm0 +; AVX-NEXT: retq + %t0 = fadd <4 x double> %x0, %x1 + %t1 = fmul <4 x double> %x2, %t0 + %t2 = fmul <4 x double> %x3, %t1 + ret <4 x double> %t2 +} + +; Verify that SSE and AVX scalar single-precision minimum ops are reassociated. + +define float @reassociate_mins_single(float %x0, float %x1, float %x2, float %x3) { +; SSE-LABEL: reassociate_mins_single: +; SSE: # BB#0: +; SSE-NEXT: divss %xmm1, %xmm0 +; SSE-NEXT: minss %xmm3, %xmm2 +; SSE-NEXT: minss %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: reassociate_mins_single: +; AVX: # BB#0: +; AVX-NEXT: vdivss %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vminss %xmm3, %xmm2, %xmm1 +; AVX-NEXT: vminss %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %t0 = fdiv float %x0, %x1 + %cmp1 = fcmp olt float %x2, %t0 + %sel1 = select i1 %cmp1, float %x2, float %t0 + %cmp2 = fcmp olt float %x3, %sel1 + %sel2 = select i1 %cmp2, float %x3, float %sel1 + ret float %sel2 +} + +; Verify that SSE and AVX scalar single-precision maximum ops are reassociated. + +define float @reassociate_maxs_single(float %x0, float %x1, float %x2, float %x3) { +; SSE-LABEL: reassociate_maxs_single: +; SSE: # BB#0: +; SSE-NEXT: divss %xmm1, %xmm0 +; SSE-NEXT: maxss %xmm3, %xmm2 +; SSE-NEXT: maxss %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: reassociate_maxs_single: +; AVX: # BB#0: +; AVX-NEXT: vdivss %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmaxss %xmm3, %xmm2, %xmm1 +; AVX-NEXT: vmaxss %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %t0 = fdiv float %x0, %x1 + %cmp1 = fcmp ogt float %x2, %t0 + %sel1 = select i1 %cmp1, float %x2, float %t0 + %cmp2 = fcmp ogt float %x3, %sel1 + %sel2 = select i1 %cmp2, float %x3, float %sel1 + ret float %sel2 +} + +; Verify that SSE and AVX scalar double-precision minimum ops are reassociated. + +define double @reassociate_mins_double(double %x0, double %x1, double %x2, double %x3) { +; SSE-LABEL: reassociate_mins_double: +; SSE: # BB#0: +; SSE-NEXT: divsd %xmm1, %xmm0 +; SSE-NEXT: minsd %xmm3, %xmm2 +; SSE-NEXT: minsd %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: reassociate_mins_double: +; AVX: # BB#0: +; AVX-NEXT: vdivsd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vminsd %xmm3, %xmm2, %xmm1 +; AVX-NEXT: vminsd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %t0 = fdiv double %x0, %x1 + %cmp1 = fcmp olt double %x2, %t0 + %sel1 = select i1 %cmp1, double %x2, double %t0 + %cmp2 = fcmp olt double %x3, %sel1 + %sel2 = select i1 %cmp2, double %x3, double %sel1 + ret double %sel2 +} + +; Verify that SSE and AVX scalar double-precision maximum ops are reassociated. + +define double @reassociate_maxs_double(double %x0, double %x1, double %x2, double %x3) { +; SSE-LABEL: reassociate_maxs_double: +; SSE: # BB#0: +; SSE-NEXT: divsd %xmm1, %xmm0 +; SSE-NEXT: maxsd %xmm3, %xmm2 +; SSE-NEXT: maxsd %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: reassociate_maxs_double: +; AVX: # BB#0: +; AVX-NEXT: vdivsd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmaxsd %xmm3, %xmm2, %xmm1 +; AVX-NEXT: vmaxsd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %t0 = fdiv double %x0, %x1 + %cmp1 = fcmp ogt double %x2, %t0 + %sel1 = select i1 %cmp1, double %x2, double %t0 + %cmp2 = fcmp ogt double %x3, %sel1 + %sel2 = select i1 %cmp2, double %x3, double %sel1 + ret double %sel2 +} + +; Verify that SSE and AVX 128-bit vector single-precision minimum ops are reassociated. + +define <4 x float> @reassociate_mins_v4f32(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) { +; SSE-LABEL: reassociate_mins_v4f32: +; SSE: # BB#0: +; SSE-NEXT: addps %xmm1, %xmm0 +; SSE-NEXT: minps %xmm3, %xmm2 +; SSE-NEXT: minps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: reassociate_mins_v4f32: +; AVX: # BB#0: +; AVX-NEXT: vaddps %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vminps %xmm3, %xmm2, %xmm1 +; AVX-NEXT: vminps %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %t0 = fadd <4 x float> %x0, %x1 + %cmp1 = fcmp olt <4 x float> %x2, %t0 + %sel1 = select <4 x i1> %cmp1, <4 x float> %x2, <4 x float> %t0 + %cmp2 = fcmp olt <4 x float> %x3, %sel1 + %sel2 = select <4 x i1> %cmp2, <4 x float> %x3, <4 x float> %sel1 + ret <4 x float> %sel2 +} + +; Verify that SSE and AVX 128-bit vector single-precision maximum ops are reassociated. + +define <4 x float> @reassociate_maxs_v4f32(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, <4 x float> %x3) { +; SSE-LABEL: reassociate_maxs_v4f32: +; SSE: # BB#0: +; SSE-NEXT: addps %xmm1, %xmm0 +; SSE-NEXT: maxps %xmm3, %xmm2 +; SSE-NEXT: maxps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: reassociate_maxs_v4f32: +; AVX: # BB#0: +; AVX-NEXT: vaddps %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmaxps %xmm3, %xmm2, %xmm1 +; AVX-NEXT: vmaxps %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %t0 = fadd <4 x float> %x0, %x1 + %cmp1 = fcmp ogt <4 x float> %x2, %t0 + %sel1 = select <4 x i1> %cmp1, <4 x float> %x2, <4 x float> %t0 + %cmp2 = fcmp ogt <4 x float> %x3, %sel1 + %sel2 = select <4 x i1> %cmp2, <4 x float> %x3, <4 x float> %sel1 + ret <4 x float> %sel2 +} + +; Verify that SSE and AVX 128-bit vector double-precision minimum ops are reassociated. + +define <2 x double> @reassociate_mins_v2f64(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, <2 x double> %x3) { +; SSE-LABEL: reassociate_mins_v2f64: +; SSE: # BB#0: +; SSE-NEXT: addpd %xmm1, %xmm0 +; SSE-NEXT: minpd %xmm3, %xmm2 +; SSE-NEXT: minpd %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: reassociate_mins_v2f64: +; AVX: # BB#0: +; AVX-NEXT: vaddpd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vminpd %xmm3, %xmm2, %xmm1 +; AVX-NEXT: vminpd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %t0 = fadd <2 x double> %x0, %x1 + %cmp1 = fcmp olt <2 x double> %x2, %t0 + %sel1 = select <2 x i1> %cmp1, <2 x double> %x2, <2 x double> %t0 + %cmp2 = fcmp olt <2 x double> %x3, %sel1 + %sel2 = select <2 x i1> %cmp2, <2 x double> %x3, <2 x double> %sel1 + ret <2 x double> %sel2 +} + +; Verify that SSE and AVX 128-bit vector double-precision maximum ops are reassociated. + +define <2 x double> @reassociate_maxs_v2f64(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, <2 x double> %x3) { +; SSE-LABEL: reassociate_maxs_v2f64: +; SSE: # BB#0: +; SSE-NEXT: addpd %xmm1, %xmm0 +; SSE-NEXT: maxpd %xmm3, %xmm2 +; SSE-NEXT: maxpd %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: reassociate_maxs_v2f64: +; AVX: # BB#0: +; AVX-NEXT: vaddpd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmaxpd %xmm3, %xmm2, %xmm1 +; AVX-NEXT: vmaxpd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %t0 = fadd <2 x double> %x0, %x1 + %cmp1 = fcmp ogt <2 x double> %x2, %t0 + %sel1 = select <2 x i1> %cmp1, <2 x double> %x2, <2 x double> %t0 + %cmp2 = fcmp ogt <2 x double> %x3, %sel1 + %sel2 = select <2 x i1> %cmp2, <2 x double> %x3, <2 x double> %sel1 + ret <2 x double> %sel2 +} + +; Verify that AVX 256-bit vector single-precision minimum ops are reassociated. + +define <8 x float> @reassociate_mins_v8f32(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, <8 x float> %x3) { +; AVX-LABEL: reassociate_mins_v8f32: +; AVX: # BB#0: +; AVX-NEXT: vaddps %ymm1, %ymm0, %ymm0 +; AVX-NEXT: vminps %ymm3, %ymm2, %ymm1 +; AVX-NEXT: vminps %ymm1, %ymm0, %ymm0 +; AVX-NEXT: retq + %t0 = fadd <8 x float> %x0, %x1 + %cmp1 = fcmp olt <8 x float> %x2, %t0 + %sel1 = select <8 x i1> %cmp1, <8 x float> %x2, <8 x float> %t0 + %cmp2 = fcmp olt <8 x float> %x3, %sel1 + %sel2 = select <8 x i1> %cmp2, <8 x float> %x3, <8 x float> %sel1 + ret <8 x float> %sel2 +} + +; Verify that AVX 256-bit vector single-precision maximum ops are reassociated. + +define <8 x float> @reassociate_maxs_v8f32(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, <8 x float> %x3) { +; AVX-LABEL: reassociate_maxs_v8f32: +; AVX: # BB#0: +; AVX-NEXT: vaddps %ymm1, %ymm0, %ymm0 +; AVX-NEXT: vmaxps %ymm3, %ymm2, %ymm1 +; AVX-NEXT: vmaxps %ymm1, %ymm0, %ymm0 +; AVX-NEXT: retq + %t0 = fadd <8 x float> %x0, %x1 + %cmp1 = fcmp ogt <8 x float> %x2, %t0 + %sel1 = select <8 x i1> %cmp1, <8 x float> %x2, <8 x float> %t0 + %cmp2 = fcmp ogt <8 x float> %x3, %sel1 + %sel2 = select <8 x i1> %cmp2, <8 x float> %x3, <8 x float> %sel1 + ret <8 x float> %sel2 +} + +; Verify that AVX 256-bit vector double-precision minimum ops are reassociated. + +define <4 x double> @reassociate_mins_v4f64(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, <4 x double> %x3) { +; AVX-LABEL: reassociate_mins_v4f64: +; AVX: # BB#0: +; AVX-NEXT: vaddpd %ymm1, %ymm0, %ymm0 +; AVX-NEXT: vminpd %ymm3, %ymm2, %ymm1 +; AVX-NEXT: vminpd %ymm1, %ymm0, %ymm0 +; AVX-NEXT: retq + %t0 = fadd <4 x double> %x0, %x1 + %cmp1 = fcmp olt <4 x double> %x2, %t0 + %sel1 = select <4 x i1> %cmp1, <4 x double> %x2, <4 x double> %t0 + %cmp2 = fcmp olt <4 x double> %x3, %sel1 + %sel2 = select <4 x i1> %cmp2, <4 x double> %x3, <4 x double> %sel1 + ret <4 x double> %sel2 +} + +; Verify that AVX 256-bit vector double-precision maximum ops are reassociated. + +define <4 x double> @reassociate_maxs_v4f64(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, <4 x double> %x3) { +; AVX-LABEL: reassociate_maxs_v4f64: +; AVX: # BB#0: +; AVX-NEXT: vaddpd %ymm1, %ymm0, %ymm0 +; AVX-NEXT: vmaxpd %ymm3, %ymm2, %ymm1 +; AVX-NEXT: vmaxpd %ymm1, %ymm0, %ymm0 +; AVX-NEXT: retq + %t0 = fadd <4 x double> %x0, %x1 + %cmp1 = fcmp ogt <4 x double> %x2, %t0 + %sel1 = select <4 x i1> %cmp1, <4 x double> %x2, <4 x double> %t0 + %cmp2 = fcmp ogt <4 x double> %x3, %sel1 + %sel2 = select <4 x i1> %cmp2, <4 x double> %x3, <4 x double> %sel1 + ret <4 x double> %sel2 +} + +; PR25016: https://llvm.org/bugs/show_bug.cgi?id=25016 +; Verify that reassociation is not happening needlessly or wrongly. + +declare double @bar() + +define double @reassociate_adds_from_calls() { +; AVX-LABEL: reassociate_adds_from_calls: +; AVX: callq bar +; AVX-NEXT: vmovsd %xmm0, 16(%rsp) +; AVX-NEXT: callq bar +; AVX-NEXT: vmovsd %xmm0, 8(%rsp) +; AVX-NEXT: callq bar +; AVX-NEXT: vmovsd %xmm0, (%rsp) +; AVX-NEXT: callq bar +; AVX-NEXT: vmovsd 8(%rsp), %xmm1 +; AVX: vaddsd 16(%rsp), %xmm1, %xmm1 +; AVX-NEXT: vaddsd (%rsp), %xmm0, %xmm0 +; AVX-NEXT: vaddsd %xmm0, %xmm1, %xmm0 + + %x0 = call double @bar() + %x1 = call double @bar() + %x2 = call double @bar() + %x3 = call double @bar() + %t0 = fadd double %x0, %x1 + %t1 = fadd double %t0, %x2 + %t2 = fadd double %t1, %x3 + ret double %t2 +} + +define double @already_reassociated() { +; AVX-LABEL: already_reassociated: +; AVX: callq bar +; AVX-NEXT: vmovsd %xmm0, 16(%rsp) +; AVX-NEXT: callq bar +; AVX-NEXT: vmovsd %xmm0, 8(%rsp) +; AVX-NEXT: callq bar +; AVX-NEXT: vmovsd %xmm0, (%rsp) +; AVX-NEXT: callq bar +; AVX-NEXT: vmovsd 8(%rsp), %xmm1 +; AVX: vaddsd 16(%rsp), %xmm1, %xmm1 +; AVX-NEXT: vaddsd (%rsp), %xmm0, %xmm0 +; AVX-NEXT: vaddsd %xmm0, %xmm1, %xmm0 + + %x0 = call double @bar() + %x1 = call double @bar() + %x2 = call double @bar() + %x3 = call double @bar() + %t0 = fadd double %x0, %x1 + %t1 = fadd double %x2, %x3 + %t2 = fadd double %t0, %t1 + ret double %t2 +} + diff --git a/test/CodeGen/X86/machine-cp.ll b/test/CodeGen/X86/machine-cp.ll index aaed0f0a23dc..143a1c3787a0 100644 --- a/test/CodeGen/X86/machine-cp.ll +++ b/test/CodeGen/X86/machine-cp.ll @@ -66,29 +66,23 @@ while.end: ; preds = %while.body, %entry ; ; CHECK-LABEL: foo: ; CHECK: psllw $7, -; CHECK: psllw $7, -; CHECK-NEXT: pand -; CHECK-NEXT: pcmpgtb -; CHECK-NEXT: pand %xmm{{[0-9]+}}, [[SRC:%xmm[0-9]+]] -; Machine propagation used to delete the first copy as the -; first few uses were . -; CHECK-NEXT: movdqa [[SRC]], [[CPY1:%xmm[0-9]+]] -; CHECK-NEXT: movdqa [[SRC]], [[CPY2:%xmm[0-9]+]] -; CHECK-NEXT: punpckhbw [[SRC]], -; Check that CPY1 is not redefined. -; CHECK-NOT: , [[CPY1]] -; undef use, we do not care. -; CHECK: punpcklwd [[CPY1]], -; Check that CPY1 is not redefined. -; CHECK-NOT: , [[CPY1]] -; CHECK: punpcklbw [[CPY2]], [[CPY2]] -; CHECK-NEXT: punpckhwd [[CPY2]], [[CPY2]] -; CHECK-NEXT pslld $31, [[CPY2]] +; CHECK: psllw $7, [[SRC1:%xmm[0-9]+]] +; CHECK-NEXT: pand {{.*}}(%rip), [[SRC1]] +; CHECK-NEXT: pcmpgtb [[SRC1]], [[SRC2:%xmm[0-9]+]] +; CHECK-NEXT: pand %xmm{{[0-9]+}}, [[SRC2]] +; CHECK-NEXT: movdqa [[SRC2]], [[CPY1:%xmm[0-9]+]] +; CHECK-NEXT: punpcklbw %xmm{{[0-9]+}}, [[CPY1]] ; Check that CPY1 is not redefined. -; CHECK-NOT: , [[CPY1]] -; CHECK: punpcklbw [[CPY1]], [[CPY1]] -; CHECK-NEXT: punpcklwd [[CPY1]], [[CPY1]] -; CHECK-NEXT pslld $31, [[CPY1]] +; CHECK-NOT: , [[CPY1]] +; CHECK: punpckhwd %xmm{{[0-9]+}}, [[CPY1]] +; CHECK-NEXT: pslld $31, [[CPY1]] +; CHECK-NEXT: psrad $31, [[CPY1]] +; CHECK: punpckhbw %xmm{{[0-9]+}}, [[CPY2:%xmm[0-9]+]] +; Check that CPY2 is not redefined. +; CHECK-NOT: , [[CPY2]] +; CHECK: punpckhwd %xmm{{[0-9]+}}, [[CPY2]] +; CHECK-NEXT: pslld $31, [[CPY2]] +; CHECK-NEXT: psrad $31, [[CPY2]] define <16 x float> @foo(<16 x float> %x) { bb: %v3 = icmp slt <16 x i32> undef, zeroinitializer diff --git a/test/CodeGen/X86/machine-trace-metrics-crash.ll b/test/CodeGen/X86/machine-trace-metrics-crash.ll index 1d0ee79f04a9..048260c51fe3 100644 --- a/test/CodeGen/X86/machine-trace-metrics-crash.ll +++ b/test/CodeGen/X86/machine-trace-metrics-crash.ll @@ -54,9 +54,9 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, isOptimized: true, runtimeVersion: 0, emissionKind: 1) !1 = !DIFile(filename: "24199.cpp", directory: "/bin") !2 = !{i32 2, !"Debug Info Version", i32 3} -!3 = !DISubprogram(linkageName: "foo", file: !1, line: 18, isLocal: false, isDefinition: true, scopeLine: 18, function: void (%struct.A*)* @foo) +!3 = distinct !DISubprogram(linkageName: "foo", file: !1, line: 18, isLocal: false, isDefinition: true, scopeLine: 18) !4 = !DIExpression() -!5 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, scope: !3, flags: DIFlagArtificial | DIFlagObjectPointer) +!5 = !DILocalVariable(name: "this", arg: 1, scope: !3, flags: DIFlagArtificial | DIFlagObjectPointer) !6 = !DILocation(line: 0, scope: !3) diff --git a/test/CodeGen/X86/masked_gather_scatter.ll b/test/CodeGen/X86/masked_gather_scatter.ll index de16e5ddc06b..b7280d87d3b7 100644 --- a/test/CodeGen/X86/masked_gather_scatter.ll +++ b/test/CodeGen/X86/masked_gather_scatter.ll @@ -1,19 +1,51 @@ -; RUN: llc -mtriple=x86_64-apple-darwin -mcpu=knl < %s | FileCheck %s -check-prefix=KNL +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f < %s | FileCheck %s --check-prefix=KNL_64 +; RUN: llc -mtriple=i386-unknown-linux-gnu -mattr=+avx512f < %s | FileCheck %s --check-prefix=KNL_32 +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl -mattr=+avx512dq < %s | FileCheck %s --check-prefix=SKX +; RUN: llc -mtriple=i386-unknown-linux-gnu -mattr=+avx512vl -mattr=+avx512dq < %s | FileCheck %s --check-prefix=SKX_32 +; RUN: opt -mtriple=x86_64-apple-darwin -codegenprepare -mcpu=corei7-avx -S < %s | FileCheck %s -check-prefix=SCALAR + target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" -; KNL-LABEL: test1 -; KNL: kxnorw %k1, %k1, %k1 -; KNL: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1} + +; SCALAR-LABEL: test1 +; SCALAR: extractelement <16 x float*> +; SCALAR-NEXT: load float +; SCALAR-NEXT: insertelement <16 x float> +; SCALAR-NEXT: extractelement <16 x float*> +; SCALAR-NEXT: load float + define <16 x float> @test1(float* %base, <16 x i32> %ind) { +; KNL_64-LABEL: test1: +; KNL_64: # BB#0: +; KNL_64-NEXT: kxnorw %k0, %k0, %k1 +; KNL_64-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1} +; KNL_64-NEXT: vmovaps %zmm1, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test1: +; KNL_32: # BB#0: +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: kxnorw %k0, %k0, %k1 +; KNL_32-NEXT: vgatherdps (%eax,%zmm0,4), %zmm1 {%k1} +; KNL_32-NEXT: vmovaps %zmm1, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test1: +; SKX: # BB#0: +; SKX-NEXT: kxnorw %k0, %k0, %k1 +; SKX-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1} +; SKX-NEXT: vmovaps %zmm1, %zmm0 +; SKX-NEXT: retq %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0 %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer %sext_ind = sext <16 x i32> %ind to <16 x i64> %gep.random = getelementptr float, <16 x float*> %broadcast.splat, <16 x i64> %sext_ind - + %res = call <16 x float> @llvm.masked.gather.v16f32(<16 x float*> %gep.random, i32 4, <16 x i1> , <16 x float> undef) ret <16 x float>%res } @@ -21,11 +53,41 @@ define <16 x float> @test1(float* %base, <16 x i32> %ind) { declare <16 x i32> @llvm.masked.gather.v16i32(<16 x i32*>, i32, <16 x i1>, <16 x i32>) declare <16 x float> @llvm.masked.gather.v16f32(<16 x float*>, i32, <16 x i1>, <16 x float>) declare <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> , i32, <8 x i1> , <8 x i32> ) - -; KNL-LABEL: test2 -; KNL: kmovw %esi, %k1 -; KNL: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1} + + +; SCALAR-LABEL: test2 +; SCALAR: extractelement <16 x float*> +; SCALAR-NEXT: load float +; SCALAR-NEXT: insertelement <16 x float> +; SCALAR-NEXT: br label %else +; SCALAR: else: +; SCALAR-NEXT: %res.phi.else = phi +; SCALAR-NEXT: %Mask1 = extractelement <16 x i1> %imask, i32 1 +; SCALAR-NEXT: %ToLoad1 = icmp eq i1 %Mask1, true +; SCALAR-NEXT: br i1 %ToLoad1, label %cond.load1, label %else2 + define <16 x float> @test2(float* %base, <16 x i32> %ind, i16 %mask) { +; KNL_64-LABEL: test2: +; KNL_64: # BB#0: +; KNL_64-NEXT: kmovw %esi, %k1 +; KNL_64-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1} +; KNL_64-NEXT: vmovaps %zmm1, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test2: +; KNL_32: # BB#0: +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: kmovw {{[0-9]+}}(%esp), %k1 +; KNL_32-NEXT: vgatherdps (%eax,%zmm0,4), %zmm1 {%k1} +; KNL_32-NEXT: vmovaps %zmm1, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test2: +; SKX: # BB#0: +; SKX-NEXT: kmovw %esi, %k1 +; SKX-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1} +; SKX-NEXT: vmovaps %zmm1, %zmm0 +; SKX-NEXT: retq %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0 %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer @@ -37,10 +99,28 @@ define <16 x float> @test2(float* %base, <16 x i32> %ind, i16 %mask) { ret <16 x float> %res } -; KNL-LABEL: test3 -; KNL: kmovw %esi, %k1 -; KNL: vpgatherdd (%rdi,%zmm0,4), %zmm1 {%k1} define <16 x i32> @test3(i32* %base, <16 x i32> %ind, i16 %mask) { +; KNL_64-LABEL: test3: +; KNL_64: # BB#0: +; KNL_64-NEXT: kmovw %esi, %k1 +; KNL_64-NEXT: vpgatherdd (%rdi,%zmm0,4), %zmm1 {%k1} +; KNL_64-NEXT: vmovaps %zmm1, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test3: +; KNL_32: # BB#0: +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: kmovw {{[0-9]+}}(%esp), %k1 +; KNL_32-NEXT: vpgatherdd (%eax,%zmm0,4), %zmm1 {%k1} +; KNL_32-NEXT: vmovaps %zmm1, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test3: +; SKX: # BB#0: +; SKX-NEXT: kmovw %esi, %k1 +; SKX-NEXT: vpgatherdd (%rdi,%zmm0,4), %zmm1 {%k1} +; SKX-NEXT: vmovaps %zmm1, %zmm0 +; SKX-NEXT: retq %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0 %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer @@ -52,13 +132,38 @@ define <16 x i32> @test3(i32* %base, <16 x i32> %ind, i16 %mask) { ret <16 x i32> %res } -; KNL-LABEL: test4 -; KNL: kmovw %esi, %k1 -; KNL: kmovw -; KNL: vpgatherdd -; KNL: vpgatherdd define <16 x i32> @test4(i32* %base, <16 x i32> %ind, i16 %mask) { +; KNL_64-LABEL: test4: +; KNL_64: # BB#0: +; KNL_64-NEXT: kmovw %esi, %k1 +; KNL_64-NEXT: kmovw %k1, %k2 +; KNL_64-NEXT: vpgatherdd (%rdi,%zmm0,4), %zmm1 {%k2} +; KNL_64-NEXT: vmovaps %zmm1, %zmm2 +; KNL_64-NEXT: vpgatherdd (%rdi,%zmm0,4), %zmm2 {%k1} +; KNL_64-NEXT: vpaddd %zmm2, %zmm1, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test4: +; KNL_32: # BB#0: +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: kmovw {{[0-9]+}}(%esp), %k1 +; KNL_32-NEXT: kmovw %k1, %k2 +; KNL_32-NEXT: vpgatherdd (%eax,%zmm0,4), %zmm1 {%k2} +; KNL_32-NEXT: vmovaps %zmm1, %zmm2 +; KNL_32-NEXT: vpgatherdd (%eax,%zmm0,4), %zmm2 {%k1} +; KNL_32-NEXT: vpaddd %zmm2, %zmm1, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test4: +; SKX: # BB#0: +; SKX-NEXT: kmovw %esi, %k1 +; SKX-NEXT: kmovw %k1, %k2 +; SKX-NEXT: vpgatherdd (%rdi,%zmm0,4), %zmm1 {%k2} +; SKX-NEXT: vmovaps %zmm1, %zmm2 +; SKX-NEXT: vpgatherdd (%rdi,%zmm0,4), %zmm2 {%k1} +; SKX-NEXT: vpaddd %zmm2, %zmm1, %zmm0 +; SKX-NEXT: retq %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0 %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer @@ -71,12 +176,46 @@ define <16 x i32> @test4(i32* %base, <16 x i32> %ind, i16 %mask) { ret <16 x i32> %res } -; KNL-LABEL: test5 -; KNL: kmovw %k1, %k2 -; KNL: vpscatterdd {{.*}}%k2 -; KNL: vpscatterdd {{.*}}%k1 + +; SCALAR-LABEL: test5 +; SCALAR: %Mask0 = extractelement <16 x i1> %imask, i32 0 +; SCALAR-NEXT: %ToStore0 = icmp eq i1 %Mask0, true +; SCALAR-NEXT: br i1 %ToStore0, label %cond.store, label %else +; SCALAR: cond.store: +; SCALAR-NEXT: %Elt0 = extractelement <16 x i32> %val, i32 0 +; SCALAR-NEXT: %Ptr0 = extractelement <16 x i32*> %gep.random, i32 0 +; SCALAR-NEXT: store i32 %Elt0, i32* %Ptr0, align 4 +; SCALAR-NEXT: br label %else +; SCALAR: else: +; SCALAR-NEXT: %Mask1 = extractelement <16 x i1> %imask, i32 1 +; SCALAR-NEXT: %ToStore1 = icmp eq i1 %Mask1, true +; SCALAR-NEXT: br i1 %ToStore1, label %cond.store1, label %else2 define void @test5(i32* %base, <16 x i32> %ind, i16 %mask, <16 x i32>%val) { +; KNL_64-LABEL: test5: +; KNL_64: # BB#0: +; KNL_64-NEXT: kmovw %esi, %k1 +; KNL_64-NEXT: kmovw %k1, %k2 +; KNL_64-NEXT: vpscatterdd %zmm1, (%rdi,%zmm0,4) {%k2} +; KNL_64-NEXT: vpscatterdd %zmm1, (%rdi,%zmm0,4) {%k1} +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test5: +; KNL_32: # BB#0: +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: kmovw {{[0-9]+}}(%esp), %k1 +; KNL_32-NEXT: kmovw %k1, %k2 +; KNL_32-NEXT: vpscatterdd %zmm1, (%eax,%zmm0,4) {%k2} +; KNL_32-NEXT: vpscatterdd %zmm1, (%eax,%zmm0,4) {%k1} +; KNL_32-NEXT: retl +; +; SKX-LABEL: test5: +; SKX: # BB#0: +; SKX-NEXT: kmovw %esi, %k1 +; SKX-NEXT: kmovw %k1, %k2 +; SKX-NEXT: vpscatterdd %zmm1, (%rdi,%zmm0,4) {%k2} +; SKX-NEXT: vpscatterdd %zmm1, (%rdi,%zmm0,4) {%k1} +; SKX-NEXT: retq %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0 %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer @@ -91,12 +230,44 @@ define void @test5(i32* %base, <16 x i32> %ind, i16 %mask, <16 x i32>%val) { declare void @llvm.masked.scatter.v8i32(<8 x i32> , <8 x i32*> , i32 , <8 x i1> ) declare void @llvm.masked.scatter.v16i32(<16 x i32> , <16 x i32*> , i32 , <16 x i1> ) -; KNL-LABEL: test6 -; KNL: kxnorw %k1, %k1, %k1 -; KNL: kxnorw %k2, %k2, %k2 -; KNL: vpgatherqd (,%zmm{{.*}}), %ymm{{.*}} {%k2} -; KNL: vpscatterqd %ymm{{.*}}, (,%zmm{{.*}}) {%k1} + +; SCALAR-LABEL: test6 +; SCALAR: store i32 %Elt0, i32* %Ptr01, align 4 +; SCALAR-NEXT: %Elt1 = extractelement <8 x i32> %a1, i32 1 +; SCALAR-NEXT: %Ptr12 = extractelement <8 x i32*> %ptr, i32 1 +; SCALAR-NEXT: store i32 %Elt1, i32* %Ptr12, align 4 +; SCALAR-NEXT: %Elt2 = extractelement <8 x i32> %a1, i32 2 +; SCALAR-NEXT: %Ptr23 = extractelement <8 x i32*> %ptr, i32 2 +; SCALAR-NEXT: store i32 %Elt2, i32* %Ptr23, align 4 + define <8 x i32> @test6(<8 x i32>%a1, <8 x i32*> %ptr) { +; KNL_64-LABEL: test6: +; KNL_64: # BB#0: +; KNL_64-NEXT: kxnorw %k0, %k0, %k1 +; KNL_64-NEXT: kxnorw %k0, %k0, %k2 +; KNL_64-NEXT: vpgatherqd (,%zmm1), %ymm2 {%k2} +; KNL_64-NEXT: vpscatterqd %ymm0, (,%zmm1) {%k1} +; KNL_64-NEXT: vmovaps %zmm2, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test6: +; KNL_32: # BB#0: +; KNL_32-NEXT: kxnorw %k0, %k0, %k1 +; KNL_32-NEXT: vpmovsxdq %ymm1, %zmm2 +; KNL_32-NEXT: kxnorw %k0, %k0, %k2 +; KNL_32-NEXT: vpgatherqd (,%zmm2), %ymm1 {%k2} +; KNL_32-NEXT: vpscatterqd %ymm0, (,%zmm2) {%k1} +; KNL_32-NEXT: vmovaps %zmm1, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test6: +; SKX: # BB#0: +; SKX-NEXT: kxnorw %k0, %k0, %k1 +; SKX-NEXT: kxnorw %k0, %k0, %k2 +; SKX-NEXT: vpgatherqd (,%zmm1), %ymm2 {%k2} +; SKX-NEXT: vpscatterqd %ymm0, (,%zmm1) {%k1} +; SKX-NEXT: vmovaps %zmm2, %zmm0 +; SKX-NEXT: retq %a = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %ptr, i32 4, <8 x i1> , <8 x i32> undef) @@ -104,13 +275,41 @@ define <8 x i32> @test6(<8 x i32>%a1, <8 x i32*> %ptr) { ret <8 x i32>%a } -; In this case the index should be promoted to <8 x i64> for KNL -; KNL-LABEL: test7 -; KNL: vpmovsxdq %ymm0, %zmm0 -; KNL: kmovw %k1, %k2 -; KNL: vpgatherqd {{.*}} {%k2} -; KNL: vpgatherqd {{.*}} {%k1} define <8 x i32> @test7(i32* %base, <8 x i32> %ind, i8 %mask) { +; +; KNL_64-LABEL: test7: +; KNL_64: # BB#0: +; KNL_64-NEXT: movzbl %sil, %eax +; KNL_64-NEXT: kmovw %eax, %k1 +; KNL_64-NEXT: vpmovsxdq %ymm0, %zmm0 +; KNL_64-NEXT: kmovw %k1, %k2 +; KNL_64-NEXT: vpgatherqd (%rdi,%zmm0,4), %ymm1 {%k2} +; KNL_64-NEXT: vmovaps %zmm1, %zmm2 +; KNL_64-NEXT: vpgatherqd (%rdi,%zmm0,4), %ymm2 {%k1} +; KNL_64-NEXT: vpaddd %ymm2, %ymm1, %ymm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test7: +; KNL_32: # BB#0: +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: kmovw {{[0-9]+}}(%esp), %k1 +; KNL_32-NEXT: vpmovsxdq %ymm0, %zmm0 +; KNL_32-NEXT: kmovw %k1, %k2 +; KNL_32-NEXT: vpgatherqd (%eax,%zmm0,4), %ymm1 {%k2} +; KNL_32-NEXT: vmovaps %zmm1, %zmm2 +; KNL_32-NEXT: vpgatherqd (%eax,%zmm0,4), %ymm2 {%k1} +; KNL_32-NEXT: vpaddd %ymm2, %ymm1, %ymm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test7: +; SKX: # BB#0: +; SKX-NEXT: kmovb %esi, %k1 +; SKX-NEXT: kmovw %k1, %k2 +; SKX-NEXT: vpgatherdd (%rdi,%ymm0,4), %ymm1 {%k2} +; SKX-NEXT: vmovaps %zmm1, %zmm2 +; SKX-NEXT: vpgatherdd (%rdi,%ymm0,4), %ymm2 {%k1} +; SKX-NEXT: vpaddd %ymm2, %ymm1, %ymm0 +; SKX-NEXT: retq %broadcast.splatinsert = insertelement <8 x i32*> undef, i32* %base, i32 0 %broadcast.splat = shufflevector <8 x i32*> %broadcast.splatinsert, <8 x i32*> undef, <8 x i32> zeroinitializer @@ -125,18 +324,1751 @@ define <8 x i32> @test7(i32* %base, <8 x i32> %ind, i8 %mask) { ; No uniform base in this case, index <8 x i64> contains addresses, ; each gather call will be split into two -; KNL-LABEL: test8 -; KNL: kshiftrw $8, %k1, %k2 -; KNL: vpgatherqd -; KNL: vpgatherqd -; KNL: vinserti64x4 -; KNL: vpgatherqd -; KNL: vpgatherqd -; KNL: vinserti64x4 define <16 x i32> @test8(<16 x i32*> %ptr.random, <16 x i32> %ind, i16 %mask) { +; KNL_64-LABEL: test8: +; KNL_64: # BB#0: +; KNL_64-NEXT: kmovw %edi, %k1 +; KNL_64-NEXT: kshiftrw $8, %k1, %k2 +; KNL_64-NEXT: kmovw %k2, %k3 +; KNL_64-NEXT: vpgatherqd (,%zmm1), %ymm2 {%k3} +; KNL_64-NEXT: kmovw %k1, %k3 +; KNL_64-NEXT: vpgatherqd (,%zmm0), %ymm3 {%k3} +; KNL_64-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm4 +; KNL_64-NEXT: vpgatherqd (,%zmm1), %ymm2 {%k2} +; KNL_64-NEXT: vpgatherqd (,%zmm0), %ymm3 {%k1} +; KNL_64-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm0 +; KNL_64-NEXT: vpaddd %zmm0, %zmm4, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test8: +; KNL_32: # BB#0: +; KNL_32-NEXT: kmovw {{[0-9]+}}(%esp), %k1 +; KNL_32-NEXT: kmovw %k1, %k2 +; KNL_32-NEXT: vpgatherdd (,%zmm0), %zmm1 {%k2} +; KNL_32-NEXT: vmovaps %zmm1, %zmm2 +; KNL_32-NEXT: vpgatherdd (,%zmm0), %zmm2 {%k1} +; KNL_32-NEXT: vpaddd %zmm2, %zmm1, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test8: +; SKX: # BB#0: +; SKX-NEXT: kmovw %edi, %k1 +; SKX-NEXT: kshiftrw $8, %k1, %k2 +; SKX-NEXT: kmovw %k2, %k3 +; SKX-NEXT: vpgatherqd (,%zmm1), %ymm2 {%k3} +; SKX-NEXT: kmovw %k1, %k3 +; SKX-NEXT: vpgatherqd (,%zmm0), %ymm3 {%k3} +; SKX-NEXT: vinserti32x8 $1, %ymm2, %zmm3, %zmm4 +; SKX-NEXT: vpgatherqd (,%zmm1), %ymm2 {%k2} +; SKX-NEXT: vpgatherqd (,%zmm0), %ymm3 {%k1} +; SKX-NEXT: vinserti32x8 $1, %ymm2, %zmm3, %zmm0 +; SKX-NEXT: vpaddd %zmm0, %zmm4, %zmm0 +; SKX-NEXT: retq +; +; SKX_32-LABEL: test8: +; SKX_32: # BB#0: +; SKX_32-NEXT: kmovw {{[0-9]+}}(%esp), %k1 +; SKX_32-NEXT: kmovw %k1, %k2 +; SKX_32-NEXT: vpgatherdd (,%zmm0), %zmm1 {%k2} +; SKX_32-NEXT: vmovaps %zmm1, %zmm2 +; SKX_32-NEXT: vpgatherdd (,%zmm0), %zmm2 {%k1} +; SKX_32-NEXT: vpaddd %zmm2, %zmm1, %zmm0 +; SKX_32-NEXT: retl + %imask = bitcast i16 %mask to <16 x i1> %gt1 = call <16 x i32> @llvm.masked.gather.v16i32(<16 x i32*> %ptr.random, i32 4, <16 x i1> %imask, <16 x i32>undef) %gt2 = call <16 x i32> @llvm.masked.gather.v16i32(<16 x i32*> %ptr.random, i32 4, <16 x i1> %imask, <16 x i32>%gt1) %res = add <16 x i32> %gt1, %gt2 ret <16 x i32> %res } + +%struct.RT = type { i8, [10 x [20 x i32]], i8 } +%struct.ST = type { i32, double, %struct.RT } + +; Masked gather for agregate types +; Test9 and Test10 should give the same result (scalar and vector indices in GEP) + + +define <8 x i32> @test9(%struct.ST* %base, <8 x i64> %ind1, <8 x i32>%ind5) { +; KNL_64-LABEL: test9: +; KNL_64: # BB#0: # %entry +; KNL_64-NEXT: vpbroadcastq %rdi, %zmm2 +; KNL_64-NEXT: vpmovsxdq %ymm1, %zmm1 +; KNL_64-NEXT: vpbroadcastq {{.*}}(%rip), %zmm3 +; KNL_64-NEXT: vpmuludq %zmm3, %zmm1, %zmm4 +; KNL_64-NEXT: vpsrlq $32, %zmm1, %zmm1 +; KNL_64-NEXT: vpmuludq %zmm3, %zmm1, %zmm1 +; KNL_64-NEXT: vpsllq $32, %zmm1, %zmm1 +; KNL_64-NEXT: vpaddq %zmm1, %zmm4, %zmm1 +; KNL_64-NEXT: vpbroadcastq {{.*}}(%rip), %zmm3 +; KNL_64-NEXT: vpmuludq %zmm3, %zmm0, %zmm4 +; KNL_64-NEXT: vpsrlq $32, %zmm0, %zmm0 +; KNL_64-NEXT: vpmuludq %zmm3, %zmm0, %zmm0 +; KNL_64-NEXT: vpsllq $32, %zmm0, %zmm0 +; KNL_64-NEXT: vpaddq %zmm0, %zmm4, %zmm0 +; KNL_64-NEXT: vpaddq %zmm0, %zmm2, %zmm0 +; KNL_64-NEXT: vpaddq %zmm1, %zmm0, %zmm0 +; KNL_64-NEXT: vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm1 +; KNL_64-NEXT: kxnorw %k0, %k0, %k1 +; KNL_64-NEXT: vpgatherqd (,%zmm1), %ymm0 {%k1} +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test9: +; KNL_32: # BB#0: # %entry +; KNL_32-NEXT: vpbroadcastd {{[0-9]+}}(%esp), %ymm2 +; KNL_32-NEXT: vpbroadcastd .LCPI8_0, %ymm3 +; KNL_32-NEXT: vpmulld %ymm3, %ymm1, %ymm1 +; KNL_32-NEXT: vpmovqd %zmm0, %ymm0 +; KNL_32-NEXT: vpbroadcastd .LCPI8_1, %ymm3 +; KNL_32-NEXT: vpmulld %ymm3, %ymm0, %ymm0 +; KNL_32-NEXT: vpaddd %ymm0, %ymm2, %ymm0 +; KNL_32-NEXT: vpaddd %ymm1, %ymm0, %ymm0 +; KNL_32-NEXT: vpbroadcastd .LCPI8_2, %ymm1 +; KNL_32-NEXT: vpaddd %ymm1, %ymm0, %ymm0 +; KNL_32-NEXT: vpmovsxdq %ymm0, %zmm1 +; KNL_32-NEXT: kxnorw %k0, %k0, %k1 +; KNL_32-NEXT: vpgatherqd (,%zmm1), %ymm0 {%k1} +; KNL_32-NEXT: retl +; +; SKX-LABEL: test9: +; SKX: # BB#0: # %entry +; SKX-NEXT: vpbroadcastq %rdi, %zmm2 +; SKX-NEXT: vpmullq {{.*}}(%rip){1to8}, %zmm0, %zmm0 +; SKX-NEXT: vpaddq %zmm0, %zmm2, %zmm0 +; SKX-NEXT: vpmovsxdq %ymm1, %zmm1 +; SKX-NEXT: vpmullq {{.*}}(%rip){1to8}, %zmm1, %zmm1 +; SKX-NEXT: vpaddq %zmm1, %zmm0, %zmm0 +; SKX-NEXT: vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm1 +; SKX-NEXT: kxnorw %k0, %k0, %k1 +; SKX-NEXT: vpgatherqd (,%zmm1), %ymm0 {%k1} +; SKX-NEXT: retq +entry: + %broadcast.splatinsert = insertelement <8 x %struct.ST*> undef, %struct.ST* %base, i32 0 + %broadcast.splat = shufflevector <8 x %struct.ST*> %broadcast.splatinsert, <8 x %struct.ST*> undef, <8 x i32> zeroinitializer + + %arrayidx = getelementptr %struct.ST, <8 x %struct.ST*> %broadcast.splat, <8 x i64> %ind1, <8 x i32> , <8 x i32>, <8 x i32> %ind5, <8 x i64> + %res = call <8 x i32 > @llvm.masked.gather.v8i32(<8 x i32*>%arrayidx, i32 4, <8 x i1> , <8 x i32> undef) + ret <8 x i32> %res +} + +define <8 x i32> @test10(%struct.ST* %base, <8 x i64> %i1, <8 x i32>%ind5) { +; KNL_64-LABEL: test10: +; KNL_64: # BB#0: # %entry +; KNL_64-NEXT: vpbroadcastq %rdi, %zmm2 +; KNL_64-NEXT: vpmovsxdq %ymm1, %zmm1 +; KNL_64-NEXT: vpbroadcastq {{.*}}(%rip), %zmm3 +; KNL_64-NEXT: vpmuludq %zmm3, %zmm1, %zmm4 +; KNL_64-NEXT: vpsrlq $32, %zmm1, %zmm1 +; KNL_64-NEXT: vpmuludq %zmm3, %zmm1, %zmm1 +; KNL_64-NEXT: vpsllq $32, %zmm1, %zmm1 +; KNL_64-NEXT: vpaddq %zmm1, %zmm4, %zmm1 +; KNL_64-NEXT: vpbroadcastq {{.*}}(%rip), %zmm3 +; KNL_64-NEXT: vpmuludq %zmm3, %zmm0, %zmm4 +; KNL_64-NEXT: vpsrlq $32, %zmm0, %zmm0 +; KNL_64-NEXT: vpmuludq %zmm3, %zmm0, %zmm0 +; KNL_64-NEXT: vpsllq $32, %zmm0, %zmm0 +; KNL_64-NEXT: vpaddq %zmm0, %zmm4, %zmm0 +; KNL_64-NEXT: vpaddq %zmm0, %zmm2, %zmm0 +; KNL_64-NEXT: vpaddq %zmm1, %zmm0, %zmm0 +; KNL_64-NEXT: vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm1 +; KNL_64-NEXT: kxnorw %k0, %k0, %k1 +; KNL_64-NEXT: vpgatherqd (,%zmm1), %ymm0 {%k1} +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test10: +; KNL_32: # BB#0: # %entry +; KNL_32-NEXT: vpbroadcastd {{[0-9]+}}(%esp), %ymm2 +; KNL_32-NEXT: vpbroadcastd .LCPI9_0, %ymm3 +; KNL_32-NEXT: vpmulld %ymm3, %ymm1, %ymm1 +; KNL_32-NEXT: vpmovqd %zmm0, %ymm0 +; KNL_32-NEXT: vpbroadcastd .LCPI9_1, %ymm3 +; KNL_32-NEXT: vpmulld %ymm3, %ymm0, %ymm0 +; KNL_32-NEXT: vpaddd %ymm0, %ymm2, %ymm0 +; KNL_32-NEXT: vpaddd %ymm1, %ymm0, %ymm0 +; KNL_32-NEXT: vpbroadcastd .LCPI9_2, %ymm1 +; KNL_32-NEXT: vpaddd %ymm1, %ymm0, %ymm0 +; KNL_32-NEXT: vpmovsxdq %ymm0, %zmm1 +; KNL_32-NEXT: kxnorw %k0, %k0, %k1 +; KNL_32-NEXT: vpgatherqd (,%zmm1), %ymm0 {%k1} +; KNL_32-NEXT: retl +; +; SKX-LABEL: test10: +; SKX: # BB#0: # %entry +; SKX-NEXT: vpbroadcastq %rdi, %zmm2 +; SKX-NEXT: vpmullq {{.*}}(%rip){1to8}, %zmm0, %zmm0 +; SKX-NEXT: vpaddq %zmm0, %zmm2, %zmm0 +; SKX-NEXT: vpmovsxdq %ymm1, %zmm1 +; SKX-NEXT: vpmullq {{.*}}(%rip){1to8}, %zmm1, %zmm1 +; SKX-NEXT: vpaddq %zmm1, %zmm0, %zmm0 +; SKX-NEXT: vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm1 +; SKX-NEXT: kxnorw %k0, %k0, %k1 +; SKX-NEXT: vpgatherqd (,%zmm1), %ymm0 {%k1} +; SKX-NEXT: retq +entry: + %broadcast.splatinsert = insertelement <8 x %struct.ST*> undef, %struct.ST* %base, i32 0 + %broadcast.splat = shufflevector <8 x %struct.ST*> %broadcast.splatinsert, <8 x %struct.ST*> undef, <8 x i32> zeroinitializer + + %arrayidx = getelementptr %struct.ST, <8 x %struct.ST*> %broadcast.splat, <8 x i64> %i1, i32 2, i32 1, <8 x i32> %ind5, i64 13 + %res = call <8 x i32 > @llvm.masked.gather.v8i32(<8 x i32*>%arrayidx, i32 4, <8 x i1> , <8 x i32> undef) + ret <8 x i32> %res +} + +; Splat index in GEP, requires broadcast +define <16 x float> @test11(float* %base, i32 %ind) { +; KNL_64-LABEL: test11: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpbroadcastd %esi, %zmm1 +; KNL_64-NEXT: kxnorw %k0, %k0, %k1 +; KNL_64-NEXT: vgatherdps (%rdi,%zmm1,4), %zmm0 {%k1} +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test11: +; KNL_32: # BB#0: +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: vpbroadcastd {{[0-9]+}}(%esp), %zmm1 +; KNL_32-NEXT: kxnorw %k0, %k0, %k1 +; KNL_32-NEXT: vgatherdps (%eax,%zmm1,4), %zmm0 {%k1} +; KNL_32-NEXT: retl +; +; SKX-LABEL: test11: +; SKX: # BB#0: +; SKX-NEXT: vpbroadcastd %esi, %zmm1 +; SKX-NEXT: kxnorw %k0, %k0, %k1 +; SKX-NEXT: vgatherdps (%rdi,%zmm1,4), %zmm0 {%k1} +; SKX-NEXT: retq + + %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0 + %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer + + %gep.random = getelementptr float, <16 x float*> %broadcast.splat, i32 %ind + + %res = call <16 x float> @llvm.masked.gather.v16f32(<16 x float*> %gep.random, i32 4, <16 x i1> , <16 x float> undef) + ret <16 x float>%res +} + +; We are checking the uniform base here. It is taken directly from input to vgatherdps +define <16 x float> @test12(float* %base, <16 x i32> %ind) { +; KNL_64-LABEL: test12: +; KNL_64: # BB#0: +; KNL_64-NEXT: kxnorw %k0, %k0, %k1 +; KNL_64-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1} +; KNL_64-NEXT: vmovaps %zmm1, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test12: +; KNL_32: # BB#0: +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: kxnorw %k0, %k0, %k1 +; KNL_32-NEXT: vgatherdps (%eax,%zmm0,4), %zmm1 {%k1} +; KNL_32-NEXT: vmovaps %zmm1, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test12: +; SKX: # BB#0: +; SKX-NEXT: kxnorw %k0, %k0, %k1 +; SKX-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1} +; SKX-NEXT: vmovaps %zmm1, %zmm0 +; SKX-NEXT: retq + + %sext_ind = sext <16 x i32> %ind to <16 x i64> + %gep.random = getelementptr float, float *%base, <16 x i64> %sext_ind + + %res = call <16 x float> @llvm.masked.gather.v16f32(<16 x float*> %gep.random, i32 4, <16 x i1> , <16 x float> undef) + ret <16 x float>%res +} + +; The same as the previous, but the mask is undefined +define <16 x float> @test13(float* %base, <16 x i32> %ind) { +; KNL_64-LABEL: test13: +; KNL_64: # BB#0: +; KNL_64-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1} +; KNL_64-NEXT: vmovaps %zmm1, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test13: +; KNL_32: # BB#0: +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: vgatherdps (%eax,%zmm0,4), %zmm1 {%k1} +; KNL_32-NEXT: vmovaps %zmm1, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test13: +; SKX: # BB#0: +; SKX-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1} +; SKX-NEXT: vmovaps %zmm1, %zmm0 +; SKX-NEXT: retq + + %sext_ind = sext <16 x i32> %ind to <16 x i64> + %gep.random = getelementptr float, float *%base, <16 x i64> %sext_ind + + %res = call <16 x float> @llvm.masked.gather.v16f32(<16 x float*> %gep.random, i32 4, <16 x i1> undef, <16 x float> undef) + ret <16 x float>%res +} + +; The base pointer is not splat, can't find unform base +define <16 x float> @test14(float* %base, i32 %ind, <16 x float*> %vec) { +; KNL_64-LABEL: test14: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpinsrq $1, %rdi, %xmm0, %xmm1 +; KNL_64-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0 +; KNL_64-NEXT: vpbroadcastq %xmm0, %zmm0 +; KNL_64-NEXT: vmovd %esi, %xmm1 +; KNL_64-NEXT: vpbroadcastd %xmm1, %ymm1 +; KNL_64-NEXT: vpmovsxdq %ymm1, %zmm1 +; KNL_64-NEXT: vpsllq $2, %zmm1, %zmm1 +; KNL_64-NEXT: vpaddq %zmm1, %zmm0, %zmm0 +; KNL_64-NEXT: kshiftrw $8, %k0, %k1 +; KNL_64-NEXT: vgatherqps (,%zmm0), %ymm1 {%k1} +; KNL_64-NEXT: vgatherqps (,%zmm0), %ymm2 {%k1} +; KNL_64-NEXT: vinsertf64x4 $1, %ymm1, %zmm2, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test14: +; KNL_32: # BB#0: +; KNL_32-NEXT: vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm1 +; KNL_32-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0 +; KNL_32-NEXT: vpbroadcastd %xmm0, %zmm0 +; KNL_32-NEXT: vpslld $2, {{[0-9]+}}(%esp){1to16}, %zmm1 +; KNL_32-NEXT: vpaddd %zmm1, %zmm0, %zmm1 +; KNL_32-NEXT: vgatherdps (,%zmm1), %zmm0 {%k1} +; KNL_32-NEXT: retl +; +; SKX-LABEL: test14: +; SKX: # BB#0: +; SKX-NEXT: vpinsrq $1, %rdi, %xmm0, %xmm1 +; SKX-NEXT: vinserti64x2 $0, %xmm1, %zmm0, %zmm0 +; SKX-NEXT: vpbroadcastq %xmm0, %zmm0 +; SKX-NEXT: vmovd %esi, %xmm1 +; SKX-NEXT: vpbroadcastd %xmm1, %ymm1 +; SKX-NEXT: vpmovsxdq %ymm1, %zmm1 +; SKX-NEXT: vpsllq $2, %zmm1, %zmm1 +; SKX-NEXT: vpaddq %zmm1, %zmm0, %zmm0 +; SKX-NEXT: kshiftrw $8, %k0, %k1 +; SKX-NEXT: vgatherqps (,%zmm0), %ymm1 {%k1} +; SKX-NEXT: vgatherqps (,%zmm0), %ymm2 {%k1} +; SKX-NEXT: vinsertf32x8 $1, %ymm1, %zmm2, %zmm0 +; SKX-NEXT: retq +; +; SKX_32-LABEL: test14: +; SKX_32: # BB#0: +; SKX_32-NEXT: vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm1 +; SKX_32-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0 +; SKX_32-NEXT: vpbroadcastd %xmm0, %zmm0 +; SKX_32-NEXT: vpslld $2, {{[0-9]+}}(%esp){1to16}, %zmm1 +; SKX_32-NEXT: vpaddd %zmm1, %zmm0, %zmm1 +; SKX_32-NEXT: vgatherdps (,%zmm1), %zmm0 {%k1} +; SKX_32-NEXT: retl + + %broadcast.splatinsert = insertelement <16 x float*> %vec, float* %base, i32 1 + %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer + + %gep.random = getelementptr float, <16 x float*> %broadcast.splat, i32 %ind + + %res = call <16 x float> @llvm.masked.gather.v16f32(<16 x float*> %gep.random, i32 4, <16 x i1> undef, <16 x float> undef) + ret <16 x float>%res +} + +declare <4 x float> @llvm.masked.gather.v4f32(<4 x float*>, i32, <4 x i1>, <4 x float>) +declare <4 x double> @llvm.masked.gather.v4f64(<4 x double*>, i32, <4 x i1>, <4 x double>) +declare <2 x double> @llvm.masked.gather.v2f64(<2 x double*>, i32, <2 x i1>, <2 x double>) + +; Gather smaller than existing instruction +define <4 x float> @test15(float* %base, <4 x i32> %ind, <4 x i1> %mask) { +; +; KNL_64-LABEL: test15: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpxor %ymm2, %ymm2, %ymm2 +; KNL_64-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7] +; KNL_64-NEXT: vpmovsxdq %ymm0, %zmm2 +; KNL_64-NEXT: vpmovsxdq %ymm1, %zmm0 +; KNL_64-NEXT: vpsllq $63, %zmm0, %zmm0 +; KNL_64-NEXT: vptestmq %zmm0, %zmm0, %k1 +; KNL_64-NEXT: vgatherqps (%rdi,%zmm2,4), %ymm0 {%k1} +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test15: +; KNL_32: # BB#0: +; KNL_32-NEXT: vpxor %ymm2, %ymm2, %ymm2 +; KNL_32-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7] +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: vpmovsxdq %ymm0, %zmm2 +; KNL_32-NEXT: vpmovsxdq %ymm1, %zmm0 +; KNL_32-NEXT: vpsllvq .LCPI14_0, %zmm0, %zmm0 +; KNL_32-NEXT: vptestmq %zmm0, %zmm0, %k1 +; KNL_32-NEXT: vgatherqps (%eax,%zmm2,4), %ymm0 {%k1} +; KNL_32-NEXT: retl +; +; SKX-LABEL: test15: +; SKX: # BB#0: +; SKX-NEXT: vpslld $31, %xmm1, %xmm1 +; SKX-NEXT: vpmovd2m %xmm1, %k1 +; SKX-NEXT: vgatherdps (%rdi,%xmm0,4), %xmm1 {%k1} +; SKX-NEXT: vmovaps %zmm1, %zmm0 +; SKX-NEXT: retq +; +; SKX_32-LABEL: test15: +; SKX_32: # BB#0: +; SKX_32-NEXT: vpslld $31, %xmm1, %xmm1 +; SKX_32-NEXT: vpmovd2m %xmm1, %k1 +; SKX_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; SKX_32-NEXT: vgatherdps (%eax,%xmm0,4), %xmm1 {%k1} +; SKX_32-NEXT: vmovaps %zmm1, %zmm0 +; SKX_32-NEXT: retl + + %sext_ind = sext <4 x i32> %ind to <4 x i64> + %gep.random = getelementptr float, float* %base, <4 x i64> %sext_ind + %res = call <4 x float> @llvm.masked.gather.v4f32(<4 x float*> %gep.random, i32 4, <4 x i1> %mask, <4 x float> undef) + ret <4 x float>%res +} + +; Gather smaller than existing instruction +define <4 x double> @test16(double* %base, <4 x i32> %ind, <4 x i1> %mask, <4 x double> %src0) { +; +; KNL_64-LABEL: test16: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpslld $31, %xmm1, %xmm1 +; KNL_64-NEXT: vpsrad $31, %xmm1, %xmm1 +; KNL_64-NEXT: vpmovsxdq %xmm1, %ymm1 +; KNL_64-NEXT: vpxord %zmm3, %zmm3, %zmm3 +; KNL_64-NEXT: vinserti64x4 $0, %ymm1, %zmm3, %zmm1 +; KNL_64-NEXT: vpmovsxdq %ymm0, %zmm0 +; KNL_64-NEXT: vpsllq $63, %zmm1, %zmm1 +; KNL_64-NEXT: vptestmq %zmm1, %zmm1, %k1 +; KNL_64-NEXT: vgatherqpd (%rdi,%zmm0,8), %zmm2 {%k1} +; KNL_64-NEXT: vmovaps %zmm2, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test16: +; KNL_32: # BB#0: +; KNL_32-NEXT: vpslld $31, %xmm1, %xmm1 +; KNL_32-NEXT: vpsrad $31, %xmm1, %xmm1 +; KNL_32-NEXT: vpmovsxdq %xmm1, %ymm1 +; KNL_32-NEXT: vpxord %zmm3, %zmm3, %zmm3 +; KNL_32-NEXT: vinserti64x4 $0, %ymm1, %zmm3, %zmm1 +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: vpmovsxdq %ymm0, %zmm0 +; KNL_32-NEXT: vpsllvq .LCPI15_0, %zmm1, %zmm1 +; KNL_32-NEXT: vptestmq %zmm1, %zmm1, %k1 +; KNL_32-NEXT: vgatherqpd (%eax,%zmm0,8), %zmm2 {%k1} +; KNL_32-NEXT: vmovaps %zmm2, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test16: +; SKX: # BB#0: +; SKX-NEXT: vpslld $31, %xmm1, %xmm1 +; SKX-NEXT: vpmovd2m %xmm1, %k1 +; SKX-NEXT: vgatherdpd (%rdi,%xmm0,8), %ymm2 {%k1} +; SKX-NEXT: vmovaps %zmm2, %zmm0 +; SKX-NEXT: retq +; +; SKX_32-LABEL: test16: +; SKX_32: # BB#0: +; SKX_32-NEXT: vpslld $31, %xmm1, %xmm1 +; SKX_32-NEXT: vpmovd2m %xmm1, %k1 +; SKX_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; SKX_32-NEXT: vgatherdpd (%eax,%xmm0,8), %ymm2 {%k1} +; SKX_32-NEXT: vmovaps %zmm2, %zmm0 +; SKX_32-NEXT: retl + + %sext_ind = sext <4 x i32> %ind to <4 x i64> + %gep.random = getelementptr double, double* %base, <4 x i64> %sext_ind + %res = call <4 x double> @llvm.masked.gather.v4f64(<4 x double*> %gep.random, i32 4, <4 x i1> %mask, <4 x double> %src0) + ret <4 x double>%res +} + +define <2 x double> @test17(double* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x double> %src0) { +; +; KNL_64-LABEL: test17: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpxord %zmm3, %zmm3, %zmm3 +; KNL_64-NEXT: vinserti32x4 $0, %xmm1, %zmm3, %zmm1 +; KNL_64-NEXT: vpsllq $63, %zmm1, %zmm1 +; KNL_64-NEXT: vptestmq %zmm1, %zmm1, %k1 +; KNL_64-NEXT: vgatherqpd (%rdi,%zmm0,8), %zmm2 {%k1} +; KNL_64-NEXT: vmovaps %zmm2, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test17: +; KNL_32: # BB#0: +; KNL_32-NEXT: vpxord %zmm3, %zmm3, %zmm3 +; KNL_32-NEXT: vinserti32x4 $0, %xmm1, %zmm3, %zmm1 +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: vpsllvq .LCPI16_0, %zmm1, %zmm1 +; KNL_32-NEXT: vptestmq %zmm1, %zmm1, %k1 +; KNL_32-NEXT: vgatherqpd (%eax,%zmm0,8), %zmm2 {%k1} +; KNL_32-NEXT: vmovaps %zmm2, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test17: +; SKX: # BB#0: +; SKX-NEXT: vpsllq $63, %xmm1, %xmm1 +; SKX-NEXT: vpmovq2m %xmm1, %k1 +; SKX-NEXT: vgatherqpd (%rdi,%xmm0,8), %xmm2 {%k1} +; SKX-NEXT: vmovaps %zmm2, %zmm0 +; SKX-NEXT: retq +; +; SKX_32-LABEL: test17: +; SKX_32: # BB#0: +; SKX_32-NEXT: vpsllq $63, %xmm1, %xmm1 +; SKX_32-NEXT: vpmovq2m %xmm1, %k1 +; SKX_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; SKX_32-NEXT: vgatherqpd (%eax,%xmm0,8), %xmm2 {%k1} +; SKX_32-NEXT: vmovaps %zmm2, %zmm0 +; SKX_32-NEXT: retl + + %sext_ind = sext <2 x i32> %ind to <2 x i64> + %gep.random = getelementptr double, double* %base, <2 x i64> %sext_ind + %res = call <2 x double> @llvm.masked.gather.v2f64(<2 x double*> %gep.random, i32 4, <2 x i1> %mask, <2 x double> %src0) + ret <2 x double>%res +} + +declare void @llvm.masked.scatter.v4i32(<4 x i32> , <4 x i32*> , i32 , <4 x i1> ) +declare void @llvm.masked.scatter.v4f64(<4 x double> , <4 x double*> , i32 , <4 x i1> ) +declare void @llvm.masked.scatter.v2i64(<2 x i64> , <2 x i64*> , i32 , <2 x i1> ) +declare void @llvm.masked.scatter.v2i32(<2 x i32> , <2 x i32*> , i32 , <2 x i1> ) +declare void @llvm.masked.scatter.v2f32(<2 x float> , <2 x float*> , i32 , <2 x i1> ) + +define void @test18(<4 x i32>%a1, <4 x i32*> %ptr, <4 x i1>%mask) { +; +; KNL_64-LABEL: test18: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpxor %ymm3, %ymm3, %ymm3 +; KNL_64-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7] +; KNL_64-NEXT: vpmovsxdq %ymm2, %zmm2 +; KNL_64-NEXT: vpsllq $63, %zmm2, %zmm2 +; KNL_64-NEXT: vptestmq %zmm2, %zmm2, %k1 +; KNL_64-NEXT: vpscatterqd %ymm0, (,%zmm1) {%k1} +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test18: +; KNL_32: # BB#0: +; KNL_32-NEXT: vpxor %ymm3, %ymm3, %ymm3 +; KNL_32-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7] +; KNL_32-NEXT: vpmovsxdq %ymm1, %zmm1 +; KNL_32-NEXT: vpmovsxdq %ymm2, %zmm2 +; KNL_32-NEXT: vpsllvq .LCPI17_0, %zmm2, %zmm2 +; KNL_32-NEXT: vptestmq %zmm2, %zmm2, %k1 +; KNL_32-NEXT: vpscatterqd %ymm0, (,%zmm1) {%k1} +; KNL_32-NEXT: retl +; +; SKX-LABEL: test18: +; SKX: # BB#0: +; SKX-NEXT: vpslld $31, %xmm2, %xmm2 +; SKX-NEXT: vpmovd2m %xmm2, %k1 +; SKX-NEXT: vpscatterqd %xmm0, (,%ymm1) {%k1} +; SKX-NEXT: retq +; +; SKX_32-LABEL: test18: +; SKX_32: # BB#0: +; SKX_32-NEXT: vpslld $31, %xmm2, %xmm2 +; SKX_32-NEXT: vpmovd2m %xmm2, %k1 +; SKX_32-NEXT: vpscatterdd %xmm0, (,%xmm1) {%k1} +; SKX_32-NEXT: retl + call void @llvm.masked.scatter.v4i32(<4 x i32> %a1, <4 x i32*> %ptr, i32 4, <4 x i1> %mask) + ret void +} + +define void @test19(<4 x double>%a1, double* %ptr, <4 x i1>%mask, <4 x i64> %ind) { +; +; KNL_64-LABEL: test19: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpslld $31, %xmm1, %xmm1 +; KNL_64-NEXT: vpsrad $31, %xmm1, %xmm1 +; KNL_64-NEXT: vpmovsxdq %xmm1, %ymm1 +; KNL_64-NEXT: vpxord %zmm3, %zmm3, %zmm3 +; KNL_64-NEXT: vinserti64x4 $0, %ymm1, %zmm3, %zmm1 +; KNL_64-NEXT: vpsllq $63, %zmm1, %zmm1 +; KNL_64-NEXT: vptestmq %zmm1, %zmm1, %k1 +; KNL_64-NEXT: vscatterqpd %zmm0, (%rdi,%zmm2,8) {%k1} +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test19: +; KNL_32: # BB#0: +; KNL_32-NEXT: vpslld $31, %xmm1, %xmm1 +; KNL_32-NEXT: vpsrad $31, %xmm1, %xmm1 +; KNL_32-NEXT: vpmovsxdq %xmm1, %ymm1 +; KNL_32-NEXT: vpxord %zmm3, %zmm3, %zmm3 +; KNL_32-NEXT: vinserti64x4 $0, %ymm1, %zmm3, %zmm1 +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: vpsllvq .LCPI18_0, %zmm1, %zmm1 +; KNL_32-NEXT: vptestmq %zmm1, %zmm1, %k1 +; KNL_32-NEXT: vscatterqpd %zmm0, (%eax,%zmm2,8) {%k1} +; KNL_32-NEXT: retl +; +; SKX-LABEL: test19: +; SKX: # BB#0: +; SKX-NEXT: vpslld $31, %xmm1, %xmm1 +; SKX-NEXT: vpmovd2m %xmm1, %k1 +; SKX-NEXT: vscatterqpd %ymm0, (%rdi,%ymm2,8) {%k1} +; SKX-NEXT: retq +; +; SKX_32-LABEL: test19: +; SKX_32: # BB#0: +; SKX_32-NEXT: vpslld $31, %xmm1, %xmm1 +; SKX_32-NEXT: vpmovd2m %xmm1, %k1 +; SKX_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; SKX_32-NEXT: vscatterqpd %ymm0, (%eax,%ymm2,8) {%k1} +; SKX_32-NEXT: retl + %gep = getelementptr double, double* %ptr, <4 x i64> %ind + call void @llvm.masked.scatter.v4f64(<4 x double> %a1, <4 x double*> %gep, i32 8, <4 x i1> %mask) + ret void +} + +; Data type requires widening +define void @test20(<2 x float>%a1, <2 x float*> %ptr, <2 x i1> %mask) { +; +; KNL_64-LABEL: test20: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; KNL_64-NEXT: vmovq {{.*#+}} xmm2 = xmm2[0],zero +; KNL_64-NEXT: vpxor %ymm3, %ymm3, %ymm3 +; KNL_64-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7] +; KNL_64-NEXT: vpmovsxdq %ymm2, %zmm2 +; KNL_64-NEXT: vpsllq $63, %zmm2, %zmm2 +; KNL_64-NEXT: vptestmq %zmm2, %zmm2, %k1 +; KNL_64-NEXT: vscatterqps %ymm0, (,%zmm1) {%k1} +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test20: +; KNL_32: # BB#0: +; KNL_32-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; KNL_32-NEXT: vmovq {{.*#+}} xmm2 = xmm2[0],zero +; KNL_32-NEXT: vpxor %ymm3, %ymm3, %ymm3 +; KNL_32-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7] +; KNL_32-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; KNL_32-NEXT: vpmovsxdq %ymm1, %zmm1 +; KNL_32-NEXT: vpmovsxdq %ymm2, %zmm2 +; KNL_32-NEXT: vpsllvq .LCPI19_0, %zmm2, %zmm2 +; KNL_32-NEXT: vptestmq %zmm2, %zmm2, %k1 +; KNL_32-NEXT: vscatterqps %ymm0, (,%zmm1) {%k1} +; KNL_32-NEXT: retl +; +; SKX-LABEL: test20: +; SKX: # BB#0: +; SKX-NEXT: vpsllq $63, %xmm2, %xmm2 +; SKX-NEXT: vpmovq2m %xmm2, %k0 +; SKX-NEXT: kshiftlw $2, %k0, %k0 +; SKX-NEXT: kshiftrw $2, %k0, %k1 +; SKX-NEXT: vscatterqps %xmm0, (,%ymm1) {%k1} +; SKX-NEXT: retq +; +; SKX_32-LABEL: test20: +; SKX_32: # BB#0: +; SKX_32-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; SKX_32-NEXT: vpsllq $63, %xmm2, %xmm2 +; SKX_32-NEXT: vpmovq2m %xmm2, %k0 +; SKX_32-NEXT: kshiftlw $2, %k0, %k0 +; SKX_32-NEXT: kshiftrw $2, %k0, %k1 +; SKX_32-NEXT: vscatterdps %xmm0, (,%xmm1) {%k1} +; SKX_32-NEXT: retl + call void @llvm.masked.scatter.v2f32(<2 x float> %a1, <2 x float*> %ptr, i32 4, <2 x i1> %mask) + ret void +} + +; Data type requires promotion +define void @test21(<2 x i32>%a1, <2 x i32*> %ptr, <2 x i1>%mask) { +; +; KNL_64-LABEL: test21: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpxord %zmm3, %zmm3, %zmm3 +; KNL_64-NEXT: vinserti32x4 $0, %xmm2, %zmm3, %zmm2 +; KNL_64-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; KNL_64-NEXT: vpsllq $63, %zmm2, %zmm2 +; KNL_64-NEXT: vptestmq %zmm2, %zmm2, %k1 +; KNL_64-NEXT: vpscatterqd %ymm0, (,%zmm1) {%k1} +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test21: +; KNL_32: # BB#0: +; KNL_32-NEXT: vpxord %zmm3, %zmm3, %zmm3 +; KNL_32-NEXT: vinserti32x4 $0, %xmm2, %zmm3, %zmm2 +; KNL_32-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; KNL_32-NEXT: vpsllvq .LCPI20_0, %zmm2, %zmm2 +; KNL_32-NEXT: vptestmq %zmm2, %zmm2, %k1 +; KNL_32-NEXT: vpscatterqd %ymm0, (,%zmm1) {%k1} +; KNL_32-NEXT: retl +; +; SKX-LABEL: test21: +; SKX: # BB#0: +; SKX-NEXT: vpsllq $63, %xmm2, %xmm2 +; SKX-NEXT: vpmovq2m %xmm2, %k0 +; SKX-NEXT: kshiftlw $2, %k0, %k0 +; SKX-NEXT: kshiftrw $2, %k0, %k1 +; SKX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; SKX-NEXT: vpscatterqd %xmm0, (,%ymm1) {%k1} +; SKX-NEXT: retq +; +; SKX_32-LABEL: test21: +; SKX_32: # BB#0: +; SKX_32-NEXT: vpsllq $63, %xmm2, %xmm2 +; SKX_32-NEXT: vpmovq2m %xmm2, %k0 +; SKX_32-NEXT: kshiftlw $2, %k0, %k0 +; SKX_32-NEXT: kshiftrw $2, %k0, %k1 +; SKX_32-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; SKX_32-NEXT: vpscatterqd %xmm0, (,%ymm1) {%k1} +; SKX_32-NEXT: retl + call void @llvm.masked.scatter.v2i32(<2 x i32> %a1, <2 x i32*> %ptr, i32 4, <2 x i1> %mask) + ret void +} + +; The result type requires widening +declare <2 x float> @llvm.masked.gather.v2f32(<2 x float*>, i32, <2 x i1>, <2 x float>) + +define <2 x float> @test22(float* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x float> %src0) { +; +; +; KNL_64-LABEL: test22: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; KNL_64-NEXT: vmovq {{.*#+}} xmm1 = xmm1[0],zero +; KNL_64-NEXT: vpxor %ymm3, %ymm3, %ymm3 +; KNL_64-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7] +; KNL_64-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; KNL_64-NEXT: vpmovsxdq %ymm0, %zmm0 +; KNL_64-NEXT: vpmovsxdq %ymm1, %zmm1 +; KNL_64-NEXT: vpsllq $63, %zmm1, %zmm1 +; KNL_64-NEXT: vptestmq %zmm1, %zmm1, %k1 +; KNL_64-NEXT: vgatherqps (%rdi,%zmm0,4), %ymm2 {%k1} +; KNL_64-NEXT: vmovaps %zmm2, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test22: +; KNL_32: # BB#0: +; KNL_32-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; KNL_32-NEXT: vmovq {{.*#+}} xmm1 = xmm1[0],zero +; KNL_32-NEXT: vpxor %ymm3, %ymm3, %ymm3 +; KNL_32-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7] +; KNL_32-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: vpmovsxdq %ymm0, %zmm0 +; KNL_32-NEXT: vpmovsxdq %ymm1, %zmm1 +; KNL_32-NEXT: vpsllvq .LCPI21_0, %zmm1, %zmm1 +; KNL_32-NEXT: vptestmq %zmm1, %zmm1, %k1 +; KNL_32-NEXT: vgatherqps (%eax,%zmm0,4), %ymm2 {%k1} +; KNL_32-NEXT: vmovaps %zmm2, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test22: +; SKX: # BB#0: +; SKX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; SKX-NEXT: vpsllq $63, %xmm1, %xmm1 +; SKX-NEXT: vpmovq2m %xmm1, %k0 +; SKX-NEXT: kshiftlw $2, %k0, %k0 +; SKX-NEXT: kshiftrw $2, %k0, %k1 +; SKX-NEXT: vgatherdps (%rdi,%xmm0,4), %xmm2 {%k1} +; SKX-NEXT: vmovaps %zmm2, %zmm0 +; SKX-NEXT: retq +; +; SKX_32-LABEL: test22: +; SKX_32: # BB#0: +; SKX_32-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; SKX_32-NEXT: vpsllq $63, %xmm1, %xmm1 +; SKX_32-NEXT: vpmovq2m %xmm1, %k0 +; SKX_32-NEXT: kshiftlw $2, %k0, %k0 +; SKX_32-NEXT: kshiftrw $2, %k0, %k1 +; SKX_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; SKX_32-NEXT: vgatherdps (%eax,%xmm0,4), %xmm2 {%k1} +; SKX_32-NEXT: vmovaps %zmm2, %zmm0 +; SKX_32-NEXT: retl + %sext_ind = sext <2 x i32> %ind to <2 x i64> + %gep.random = getelementptr float, float* %base, <2 x i64> %sext_ind + %res = call <2 x float> @llvm.masked.gather.v2f32(<2 x float*> %gep.random, i32 4, <2 x i1> %mask, <2 x float> %src0) + ret <2 x float>%res +} + +declare <2 x i32> @llvm.masked.gather.v2i32(<2 x i32*>, i32, <2 x i1>, <2 x i32>) +declare <2 x i64> @llvm.masked.gather.v2i64(<2 x i64*>, i32, <2 x i1>, <2 x i64>) + +define <2 x i32> @test23(i32* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x i32> %src0) { +; +; KNL_64-LABEL: test23: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpxord %zmm3, %zmm3, %zmm3 +; KNL_64-NEXT: vinserti32x4 $0, %xmm1, %zmm3, %zmm1 +; KNL_64-NEXT: vpsllq $63, %zmm1, %zmm1 +; KNL_64-NEXT: vptestmq %zmm1, %zmm1, %k1 +; KNL_64-NEXT: vpgatherqq (%rdi,%zmm0,8), %zmm2 {%k1} +; KNL_64-NEXT: vmovaps %zmm2, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test23: +; KNL_32: # BB#0: +; KNL_32-NEXT: vpxord %zmm3, %zmm3, %zmm3 +; KNL_32-NEXT: vinserti32x4 $0, %xmm1, %zmm3, %zmm1 +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: vpsllvq .LCPI22_0, %zmm1, %zmm1 +; KNL_32-NEXT: vptestmq %zmm1, %zmm1, %k1 +; KNL_32-NEXT: vpgatherqq (%eax,%zmm0,8), %zmm2 {%k1} +; KNL_32-NEXT: vmovaps %zmm2, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test23: +; SKX: # BB#0: +; SKX-NEXT: vpsllq $63, %xmm1, %xmm1 +; SKX-NEXT: vpmovq2m %xmm1, %k1 +; SKX-NEXT: vpgatherqq (%rdi,%xmm0,8), %xmm2 {%k1} +; SKX-NEXT: vmovaps %zmm2, %zmm0 +; SKX-NEXT: retq +; +; SKX_32-LABEL: test23: +; SKX_32: # BB#0: +; SKX_32-NEXT: vpsllq $63, %xmm1, %xmm1 +; SKX_32-NEXT: vpmovq2m %xmm1, %k1 +; SKX_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; SKX_32-NEXT: vpgatherqq (%eax,%xmm0,8), %xmm2 {%k1} +; SKX_32-NEXT: vmovaps %zmm2, %zmm0 +; SKX_32-NEXT: retl + %sext_ind = sext <2 x i32> %ind to <2 x i64> + %gep.random = getelementptr i32, i32* %base, <2 x i64> %sext_ind + %res = call <2 x i32> @llvm.masked.gather.v2i32(<2 x i32*> %gep.random, i32 4, <2 x i1> %mask, <2 x i32> %src0) + ret <2 x i32>%res +} + +define <2 x i32> @test24(i32* %base, <2 x i32> %ind) { +; KNL_64-LABEL: test24: +; KNL_64: # BB#0: +; KNL_64-NEXT: movb $3, %al +; KNL_64-NEXT: movzbl %al, %eax +; KNL_64-NEXT: kmovw %eax, %k1 +; KNL_64-NEXT: vpgatherqq (%rdi,%zmm0,8), %zmm1 {%k1} +; KNL_64-NEXT: vmovaps %zmm1, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test24: +; KNL_32: # BB#0: +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; KNL_32-NEXT: vinserti32x4 $0, .LCPI23_0, %zmm1, %zmm1 +; KNL_32-NEXT: vpsllvq .LCPI23_1, %zmm1, %zmm1 +; KNL_32-NEXT: vptestmq %zmm1, %zmm1, %k1 +; KNL_32-NEXT: vpgatherqq (%eax,%zmm0,8), %zmm1 {%k1} +; KNL_32-NEXT: vmovaps %zmm1, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test24: +; SKX: # BB#0: +; SKX-NEXT: kxnorw %k0, %k0, %k1 +; SKX-NEXT: vpgatherqq (%rdi,%xmm0,8), %xmm1 {%k1} +; SKX-NEXT: vmovaps %zmm1, %zmm0 +; SKX-NEXT: retq +; +; SKX_32-LABEL: test24: +; SKX_32: # BB#0: +; SKX_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; SKX_32-NEXT: kxnorw %k0, %k0, %k1 +; SKX_32-NEXT: vpgatherqq (%eax,%xmm0,8), %xmm1 {%k1} +; SKX_32-NEXT: vmovaps %zmm1, %zmm0 +; SKX_32-NEXT: retl + %sext_ind = sext <2 x i32> %ind to <2 x i64> + %gep.random = getelementptr i32, i32* %base, <2 x i64> %sext_ind + %res = call <2 x i32> @llvm.masked.gather.v2i32(<2 x i32*> %gep.random, i32 4, <2 x i1> , <2 x i32> undef) + ret <2 x i32>%res +} + +define <2 x i64> @test25(i64* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x i64> %src0) { +; +; KNL_64-LABEL: test25: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpxord %zmm3, %zmm3, %zmm3 +; KNL_64-NEXT: vinserti32x4 $0, %xmm1, %zmm3, %zmm1 +; KNL_64-NEXT: vpsllq $63, %zmm1, %zmm1 +; KNL_64-NEXT: vptestmq %zmm1, %zmm1, %k1 +; KNL_64-NEXT: vpgatherqq (%rdi,%zmm0,8), %zmm2 {%k1} +; KNL_64-NEXT: vmovaps %zmm2, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test25: +; KNL_32: # BB#0: +; KNL_32-NEXT: vpxord %zmm3, %zmm3, %zmm3 +; KNL_32-NEXT: vinserti32x4 $0, %xmm1, %zmm3, %zmm1 +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: vpsllvq .LCPI24_0, %zmm1, %zmm1 +; KNL_32-NEXT: vptestmq %zmm1, %zmm1, %k1 +; KNL_32-NEXT: vpgatherqq (%eax,%zmm0,8), %zmm2 {%k1} +; KNL_32-NEXT: vmovaps %zmm2, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test25: +; SKX: # BB#0: +; SKX-NEXT: vpsllq $63, %xmm1, %xmm1 +; SKX-NEXT: vpmovq2m %xmm1, %k1 +; SKX-NEXT: vpgatherqq (%rdi,%xmm0,8), %xmm2 {%k1} +; SKX-NEXT: vmovaps %zmm2, %zmm0 +; SKX-NEXT: retq +; +; SKX_32-LABEL: test25: +; SKX_32: # BB#0: +; SKX_32-NEXT: vpsllq $63, %xmm1, %xmm1 +; SKX_32-NEXT: vpmovq2m %xmm1, %k1 +; SKX_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; SKX_32-NEXT: vpgatherqq (%eax,%xmm0,8), %xmm2 {%k1} +; SKX_32-NEXT: vmovaps %zmm2, %zmm0 +; SKX_32-NEXT: retl + %sext_ind = sext <2 x i32> %ind to <2 x i64> + %gep.random = getelementptr i64, i64* %base, <2 x i64> %sext_ind + %res = call <2 x i64> @llvm.masked.gather.v2i64(<2 x i64*> %gep.random, i32 8, <2 x i1> %mask, <2 x i64> %src0) + ret <2 x i64>%res +} + +define <2 x i64> @test26(i64* %base, <2 x i32> %ind, <2 x i64> %src0) { +; +; KNL_64-LABEL: test26: +; KNL_64: # BB#0: +; KNL_64-NEXT: movb $3, %al +; KNL_64-NEXT: movzbl %al, %eax +; KNL_64-NEXT: kmovw %eax, %k1 +; KNL_64-NEXT: vpgatherqq (%rdi,%zmm0,8), %zmm1 {%k1} +; KNL_64-NEXT: vmovaps %zmm1, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test26: +; KNL_32: # BB#0: +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: vpxord %zmm2, %zmm2, %zmm2 +; KNL_32-NEXT: vinserti32x4 $0, .LCPI25_0, %zmm2, %zmm2 +; KNL_32-NEXT: vpsllvq .LCPI25_1, %zmm2, %zmm2 +; KNL_32-NEXT: vptestmq %zmm2, %zmm2, %k1 +; KNL_32-NEXT: vpgatherqq (%eax,%zmm0,8), %zmm1 {%k1} +; KNL_32-NEXT: vmovaps %zmm1, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test26: +; SKX: # BB#0: +; SKX-NEXT: kxnorw %k0, %k0, %k1 +; SKX-NEXT: vpgatherqq (%rdi,%xmm0,8), %xmm1 {%k1} +; SKX-NEXT: vmovaps %zmm1, %zmm0 +; SKX-NEXT: retq +; +; SKX_32-LABEL: test26: +; SKX_32: # BB#0: +; SKX_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; SKX_32-NEXT: kxnorw %k0, %k0, %k1 +; SKX_32-NEXT: vpgatherqq (%eax,%xmm0,8), %xmm1 {%k1} +; SKX_32-NEXT: vmovaps %zmm1, %zmm0 +; SKX_32-NEXT: retl + %sext_ind = sext <2 x i32> %ind to <2 x i64> + %gep.random = getelementptr i64, i64* %base, <2 x i64> %sext_ind + %res = call <2 x i64> @llvm.masked.gather.v2i64(<2 x i64*> %gep.random, i32 8, <2 x i1> , <2 x i64> %src0) + ret <2 x i64>%res +} + +; Result type requires widening; all-ones mask +define <2 x float> @test27(float* %base, <2 x i32> %ind) { +; +; KNL_64-LABEL: test27: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; KNL_64-NEXT: vpmovsxdq %ymm0, %zmm1 +; KNL_64-NEXT: movb $3, %al +; KNL_64-NEXT: movzbl %al, %eax +; KNL_64-NEXT: kmovw %eax, %k1 +; KNL_64-NEXT: vgatherqps (%rdi,%zmm1,4), %ymm0 {%k1} +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test27: +; KNL_32: # BB#0: +; KNL_32-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: vpmovsxdq %ymm0, %zmm1 +; KNL_32-NEXT: movb $3, %cl +; KNL_32-NEXT: movzbl %cl, %ecx +; KNL_32-NEXT: kmovw %ecx, %k1 +; KNL_32-NEXT: vgatherqps (%eax,%zmm1,4), %ymm0 {%k1} +; KNL_32-NEXT: retl +; +; SKX-LABEL: test27: +; SKX: # BB#0: +; SKX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,2,2,3] +; SKX-NEXT: movb $3, %al +; SKX-NEXT: kmovb %eax, %k1 +; SKX-NEXT: vgatherdps (%rdi,%xmm1,4), %xmm0 {%k1} +; SKX-NEXT: retq + %sext_ind = sext <2 x i32> %ind to <2 x i64> + %gep.random = getelementptr float, float* %base, <2 x i64> %sext_ind + %res = call <2 x float> @llvm.masked.gather.v2f32(<2 x float*> %gep.random, i32 4, <2 x i1> , <2 x float> undef) + ret <2 x float>%res +} + +; Data type requires promotion, mask is all-ones +define void @test28(<2 x i32>%a1, <2 x i32*> %ptr) { +; +; +; KNL_64-LABEL: test28: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; KNL_64-NEXT: movb $3, %al +; KNL_64-NEXT: movzbl %al, %eax +; KNL_64-NEXT: kmovw %eax, %k1 +; KNL_64-NEXT: vpscatterqd %ymm0, (,%zmm1) {%k1} +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test28: +; KNL_32: # BB#0: +; KNL_32-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; KNL_32-NEXT: vpxord %zmm2, %zmm2, %zmm2 +; KNL_32-NEXT: vinserti32x4 $0, .LCPI27_0, %zmm2, %zmm2 +; KNL_32-NEXT: vpsllvq .LCPI27_1, %zmm2, %zmm2 +; KNL_32-NEXT: vptestmq %zmm2, %zmm2, %k1 +; KNL_32-NEXT: vpscatterqd %ymm0, (,%zmm1) {%k1} +; KNL_32-NEXT: retl +; +; SKX-LABEL: test28: +; SKX: # BB#0: +; SKX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; SKX-NEXT: movb $3, %al +; SKX-NEXT: kmovb %eax, %k1 +; SKX-NEXT: vpscatterqd %xmm0, (,%ymm1) {%k1} +; SKX-NEXT: retq +; +; SKX_32-LABEL: test28: +; SKX_32: # BB#0: +; SKX_32-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; SKX_32-NEXT: movb $3, %al +; SKX_32-NEXT: kmovb %eax, %k1 +; SKX_32-NEXT: vpscatterqd %xmm0, (,%ymm1) {%k1} +; SKX_32-NEXT: retl + call void @llvm.masked.scatter.v2i32(<2 x i32> %a1, <2 x i32*> %ptr, i32 4, <2 x i1> ) + ret void +} + + +; SCALAR-LABEL: test29 +; SCALAR: extractelement <16 x float*> +; SCALAR-NEXT: load float +; SCALAR-NEXT: insertelement <16 x float> +; SCALAR-NEXT: extractelement <16 x float*> +; SCALAR-NEXT: load float + +define <16 x float> @test29(float* %base, <16 x i32> %ind) { +; KNL_64-LABEL: test29: +; KNL_64: # BB#0: +; KNL_64-NEXT: movw $44, %ax +; KNL_64-NEXT: kmovw %eax, %k1 +; KNL_64-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1} +; KNL_64-NEXT: vmovaps %zmm1, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test29: +; KNL_32: # BB#0: +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: movw $44, %cx +; KNL_32-NEXT: kmovw %ecx, %k1 +; KNL_32-NEXT: vgatherdps (%eax,%zmm0,4), %zmm1 {%k1} +; KNL_32-NEXT: vmovaps %zmm1, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test29: +; SKX: # BB#0: +; SKX-NEXT: movw $44, %ax +; SKX-NEXT: kmovw %eax, %k1 +; SKX-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1} +; SKX-NEXT: vmovaps %zmm1, %zmm0 +; SKX-NEXT: retq + + %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0 + %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer + + %sext_ind = sext <16 x i32> %ind to <16 x i64> + %gep.random = getelementptr float, <16 x float*> %broadcast.splat, <16 x i64> %sext_ind + + %res = call <16 x float> @llvm.masked.gather.v16f32(<16 x float*> %gep.random, i32 4, <16 x i1> , <16 x float> undef) + ret <16 x float>%res +} + +; Check non-power-of-2 case. It should be scalarized. +declare <3 x i32> @llvm.masked.gather.v3i32(<3 x i32*>, i32, <3 x i1>, <3 x i32>) +define <3 x i32> @test30(<3 x i32*> %base, <3 x i32> %ind, <3 x i1> %mask, <3 x i32> %src0) { +; KNL_64-LABEL: test30: +; KNL_64: # BB#0: +; KNL_64-NEXT: andl $1, %edx +; KNL_64-NEXT: kmovw %edx, %k1 +; KNL_64-NEXT: andl $1, %esi +; KNL_64-NEXT: kmovw %esi, %k2 +; KNL_64-NEXT: movl %edi, %eax +; KNL_64-NEXT: andl $1, %eax +; KNL_64-NEXT: kmovw %eax, %k0 +; KNL_64-NEXT: vpmovsxdq %xmm1, %ymm1 +; KNL_64-NEXT: vpsllq $2, %ymm1, %ymm1 +; KNL_64-NEXT: vpaddq %ymm1, %ymm0, %ymm1 +; KNL_64-NEXT: # implicit-def: %XMM0 +; KNL_64-NEXT: testb $1, %dil +; KNL_64-NEXT: je .LBB29_2 +; KNL_64-NEXT: # BB#1: # %cond.load +; KNL_64-NEXT: vmovq %xmm1, %rax +; KNL_64-NEXT: vmovd (%rax), %xmm0 +; KNL_64-NEXT: .LBB29_2: # %else +; KNL_64-NEXT: kmovw %k2, %eax +; KNL_64-NEXT: movl %eax, %ecx +; KNL_64-NEXT: andl $1, %ecx +; KNL_64-NEXT: testb %cl, %cl +; KNL_64-NEXT: je .LBB29_4 +; KNL_64-NEXT: # BB#3: # %cond.load1 +; KNL_64-NEXT: vpextrq $1, %xmm1, %rcx +; KNL_64-NEXT: vpinsrd $1, (%rcx), %xmm0, %xmm0 +; KNL_64-NEXT: .LBB29_4: # %else2 +; KNL_64-NEXT: kmovw %k1, %ecx +; KNL_64-NEXT: movl %ecx, %edx +; KNL_64-NEXT: andl $1, %edx +; KNL_64-NEXT: testb %dl, %dl +; KNL_64-NEXT: je .LBB29_6 +; KNL_64-NEXT: # BB#5: # %cond.load4 +; KNL_64-NEXT: vextracti128 $1, %ymm1, %xmm1 +; KNL_64-NEXT: vmovq %xmm1, %rdx +; KNL_64-NEXT: vpinsrd $2, (%rdx), %xmm0, %xmm0 +; KNL_64-NEXT: .LBB29_6: # %else5 +; KNL_64-NEXT: kmovw %k0, %edx +; KNL_64-NEXT: vmovd %edx, %xmm1 +; KNL_64-NEXT: vpinsrd $1, %eax, %xmm1, %xmm1 +; KNL_64-NEXT: vpinsrd $2, %ecx, %xmm1, %xmm1 +; KNL_64-NEXT: vpslld $31, %xmm1, %xmm1 +; KNL_64-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test30: +; KNL_32: # BB#0: +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: andl $1, %eax +; KNL_32-NEXT: kmovw %eax, %k1 +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: andl $1, %eax +; KNL_32-NEXT: kmovw %eax, %k2 +; KNL_32-NEXT: movl {{[0-9]+}}(%esp), %eax +; KNL_32-NEXT: movl %eax, %ecx +; KNL_32-NEXT: andl $1, %ecx +; KNL_32-NEXT: kmovw %ecx, %k0 +; KNL_32-NEXT: vpslld $2, %xmm1, %xmm1 +; KNL_32-NEXT: vpaddd %xmm1, %xmm0, %xmm1 +; KNL_32-NEXT: # implicit-def: %XMM0 +; KNL_32-NEXT: testb $1, %al +; KNL_32-NEXT: je .LBB29_2 +; KNL_32-NEXT: # BB#1: # %cond.load +; KNL_32-NEXT: vmovd %xmm1, %eax +; KNL_32-NEXT: vmovd (%eax), %xmm0 +; KNL_32-NEXT: .LBB29_2: # %else +; KNL_32-NEXT: kmovw %k2, %eax +; KNL_32-NEXT: movl %eax, %ecx +; KNL_32-NEXT: andl $1, %ecx +; KNL_32-NEXT: testb %cl, %cl +; KNL_32-NEXT: je .LBB29_4 +; KNL_32-NEXT: # BB#3: # %cond.load1 +; KNL_32-NEXT: vpextrd $1, %xmm1, %ecx +; KNL_32-NEXT: vpinsrd $1, (%ecx), %xmm0, %xmm0 +; KNL_32-NEXT: .LBB29_4: # %else2 +; KNL_32-NEXT: kmovw %k1, %ecx +; KNL_32-NEXT: movl %ecx, %edx +; KNL_32-NEXT: andl $1, %edx +; KNL_32-NEXT: testb %dl, %dl +; KNL_32-NEXT: je .LBB29_6 +; KNL_32-NEXT: # BB#5: # %cond.load4 +; KNL_32-NEXT: vpextrd $2, %xmm1, %edx +; KNL_32-NEXT: vpinsrd $2, (%edx), %xmm0, %xmm0 +; KNL_32-NEXT: .LBB29_6: # %else5 +; KNL_32-NEXT: kmovw %k0, %edx +; KNL_32-NEXT: vmovd %edx, %xmm1 +; KNL_32-NEXT: vpinsrd $1, %eax, %xmm1, %xmm1 +; KNL_32-NEXT: vpinsrd $2, %ecx, %xmm1, %xmm1 +; KNL_32-NEXT: vpslld $31, %xmm1, %xmm1 +; KNL_32-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test30: +; SKX: # BB#0: +; SKX-NEXT: vpslld $31, %xmm2, %xmm2 +; SKX-NEXT: vpmovd2m %xmm2, %k1 +; SKX-NEXT: kmovb %k1, -{{[0-9]+}}(%rsp) +; SKX-NEXT: vpmovsxdq %xmm1, %ymm1 +; SKX-NEXT: vpsllq $2, %ymm1, %ymm1 +; SKX-NEXT: vpaddq %ymm1, %ymm0, %ymm1 +; SKX-NEXT: movb -{{[0-9]+}}(%rsp), %al +; SKX-NEXT: # implicit-def: %XMM0 +; SKX-NEXT: andb $1, %al +; SKX-NEXT: je .LBB29_2 +; SKX-NEXT: # BB#1: # %cond.load +; SKX-NEXT: vmovq %xmm1, %rax +; SKX-NEXT: vmovd (%rax), %xmm0 +; SKX-NEXT: .LBB29_2: # %else +; SKX-NEXT: kmovb %k1, -{{[0-9]+}}(%rsp) +; SKX-NEXT: movb -{{[0-9]+}}(%rsp), %al +; SKX-NEXT: andb $1, %al +; SKX-NEXT: je .LBB29_4 +; SKX-NEXT: # BB#3: # %cond.load1 +; SKX-NEXT: vpextrq $1, %xmm1, %rax +; SKX-NEXT: vpinsrd $1, (%rax), %xmm0, %xmm0 +; SKX-NEXT: .LBB29_4: # %else2 +; SKX-NEXT: kmovb %k1, -{{[0-9]+}}(%rsp) +; SKX-NEXT: movb -{{[0-9]+}}(%rsp), %al +; SKX-NEXT: andb $1, %al +; SKX-NEXT: je .LBB29_6 +; SKX-NEXT: # BB#5: # %cond.load4 +; SKX-NEXT: vextracti128 $1, %ymm1, %xmm1 +; SKX-NEXT: vmovq %xmm1, %rax +; SKX-NEXT: vpinsrd $2, (%rax), %xmm0, %xmm0 +; SKX-NEXT: .LBB29_6: # %else5 +; SKX-NEXT: vmovdqa32 %xmm0, %xmm3 {%k1} +; SKX-NEXT: vmovaps %zmm3, %zmm0 +; SKX-NEXT: retq +; +; SKX_32-LABEL: test30: +; SKX_32: # BB#0: +; SKX_32-NEXT: subl $12, %esp +; SKX_32-NEXT: .Ltmp0: +; SKX_32-NEXT: .cfi_def_cfa_offset 16 +; SKX_32-NEXT: vpslld $31, %xmm2, %xmm2 +; SKX_32-NEXT: vpmovd2m %xmm2, %k1 +; SKX_32-NEXT: kmovb %k1, {{[0-9]+}}(%esp) +; SKX_32-NEXT: vpslld $2, %xmm1, %xmm1 +; SKX_32-NEXT: vpaddd %xmm1, %xmm0, %xmm2 +; SKX_32-NEXT: movb {{[0-9]+}}(%esp), %al +; SKX_32-NEXT: # implicit-def: %XMM1 +; SKX_32-NEXT: andb $1, %al +; SKX_32-NEXT: je .LBB29_2 +; SKX_32-NEXT: # BB#1: # %cond.load +; SKX_32-NEXT: vmovd %xmm2, %eax +; SKX_32-NEXT: vmovd (%eax), %xmm1 +; SKX_32-NEXT: .LBB29_2: # %else +; SKX_32-NEXT: kmovb %k1, {{[0-9]+}}(%esp) +; SKX_32-NEXT: movb {{[0-9]+}}(%esp), %al +; SKX_32-NEXT: andb $1, %al +; SKX_32-NEXT: je .LBB29_4 +; SKX_32-NEXT: # BB#3: # %cond.load1 +; SKX_32-NEXT: vpextrd $1, %xmm2, %eax +; SKX_32-NEXT: vpinsrd $1, (%eax), %xmm1, %xmm1 +; SKX_32-NEXT: .LBB29_4: # %else2 +; SKX_32-NEXT: vmovdqa32 {{[0-9]+}}(%esp), %xmm0 +; SKX_32-NEXT: kmovb %k1, (%esp) +; SKX_32-NEXT: movb (%esp), %al +; SKX_32-NEXT: andb $1, %al +; SKX_32-NEXT: je .LBB29_6 +; SKX_32-NEXT: # BB#5: # %cond.load4 +; SKX_32-NEXT: vpextrd $2, %xmm2, %eax +; SKX_32-NEXT: vpinsrd $2, (%eax), %xmm1, %xmm1 +; SKX_32-NEXT: .LBB29_6: # %else5 +; SKX_32-NEXT: vmovdqa32 %xmm1, %xmm0 {%k1} +; SKX_32-NEXT: addl $12, %esp +; SKX_32-NEXT: retl + + %sext_ind = sext <3 x i32> %ind to <3 x i64> + %gep.random = getelementptr i32, <3 x i32*> %base, <3 x i64> %sext_ind + %res = call <3 x i32> @llvm.masked.gather.v3i32(<3 x i32*> %gep.random, i32 4, <3 x i1> %mask, <3 x i32> %src0) + ret <3 x i32>%res +} + +declare <16 x float*> @llvm.masked.gather.v16p0f32(<16 x float**>, i32, <16 x i1>, <16 x float*>) + +; KNL-LABEL: test31 +; KNL: vpgatherqq +; KNL: vpgatherqq +define <16 x float*> @test31(<16 x float**> %ptrs) { +; KNL_64-LABEL: test31: +; KNL_64: # BB#0: +; KNL_64-NEXT: kxnorw %k0, %k0, %k1 +; KNL_64-NEXT: kxnorw %k0, %k0, %k2 +; KNL_64-NEXT: vpgatherqq (,%zmm0), %zmm2 {%k2} +; KNL_64-NEXT: kshiftrw $8, %k1, %k1 +; KNL_64-NEXT: vpgatherqq (,%zmm1), %zmm3 {%k1} +; KNL_64-NEXT: vmovaps %zmm2, %zmm0 +; KNL_64-NEXT: vmovaps %zmm3, %zmm1 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test31: +; KNL_32: # BB#0: +; KNL_32-NEXT: kxnorw %k0, %k0, %k1 +; KNL_32-NEXT: vpgatherdd (,%zmm0), %zmm1 {%k1} +; KNL_32-NEXT: vmovaps %zmm1, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test31: +; SKX: # BB#0: +; SKX-NEXT: kxnorw %k0, %k0, %k1 +; SKX-NEXT: kxnorw %k0, %k0, %k2 +; SKX-NEXT: vpgatherqq (,%zmm0), %zmm2 {%k2} +; SKX-NEXT: kshiftrw $8, %k1, %k1 +; SKX-NEXT: vpgatherqq (,%zmm1), %zmm3 {%k1} +; SKX-NEXT: vmovaps %zmm2, %zmm0 +; SKX-NEXT: vmovaps %zmm3, %zmm1 +; SKX-NEXT: retq +; +; SKX_32-LABEL: test31: +; SKX_32: # BB#0: +; SKX_32-NEXT: kxnorw %k0, %k0, %k1 +; SKX_32-NEXT: vpgatherdd (,%zmm0), %zmm1 {%k1} +; SKX_32-NEXT: vmovaps %zmm1, %zmm0 +; SKX_32-NEXT: retl + + %res = call <16 x float*> @llvm.masked.gather.v16p0f32(<16 x float**> %ptrs, i32 4, <16 x i1> , <16 x float*> undef) + ret <16 x float*>%res +} + +define <16 x i32> @test_gather_16i32(<16 x i32*> %ptrs, <16 x i1> %mask, <16 x i32> %src0) { +; KNL_64-LABEL: test_gather_16i32: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpmovsxbd %xmm2, %zmm2 +; KNL_64-NEXT: vpslld $31, %zmm2, %zmm2 +; KNL_64-NEXT: vptestmd %zmm2, %zmm2, %k1 +; KNL_64-NEXT: vextracti64x4 $1, %zmm3, %ymm2 +; KNL_64-NEXT: kshiftrw $8, %k1, %k2 +; KNL_64-NEXT: vpgatherqd (,%zmm1), %ymm2 {%k2} +; KNL_64-NEXT: vpgatherqd (,%zmm0), %ymm3 {%k1} +; KNL_64-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test_gather_16i32: +; KNL_32: # BB#0: +; KNL_32-NEXT: vpmovsxbd %xmm1, %zmm1 +; KNL_32-NEXT: vpslld $31, %zmm1, %zmm1 +; KNL_32-NEXT: vptestmd %zmm1, %zmm1, %k1 +; KNL_32-NEXT: vpgatherdd (,%zmm0), %zmm2 {%k1} +; KNL_32-NEXT: vmovaps %zmm2, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test_gather_16i32: +; SKX: # BB#0: +; SKX-NEXT: vpmovsxbd %xmm2, %zmm2 +; SKX-NEXT: vpslld $31, %zmm2, %zmm2 +; SKX-NEXT: vptestmd %zmm2, %zmm2, %k1 +; SKX-NEXT: vextracti32x8 $1, %zmm3, %ymm2 +; SKX-NEXT: kshiftrw $8, %k1, %k2 +; SKX-NEXT: vpgatherqd (,%zmm1), %ymm2 {%k2} +; SKX-NEXT: vpgatherqd (,%zmm0), %ymm3 {%k1} +; SKX-NEXT: vinserti32x8 $1, %ymm2, %zmm3, %zmm0 +; SKX-NEXT: retq +; +; SKX_32-LABEL: test_gather_16i32: +; SKX_32: # BB#0: +; SKX_32-NEXT: vpmovsxbd %xmm1, %zmm1 +; SKX_32-NEXT: vpslld $31, %zmm1, %zmm1 +; SKX_32-NEXT: vptestmd %zmm1, %zmm1, %k1 +; SKX_32-NEXT: vpgatherdd (,%zmm0), %zmm2 {%k1} +; SKX_32-NEXT: vmovaps %zmm2, %zmm0 +; SKX_32-NEXT: retl + %res = call <16 x i32> @llvm.masked.gather.v16i32(<16 x i32*> %ptrs, i32 4, <16 x i1> %mask, <16 x i32> %src0) + ret <16 x i32> %res +} +define <16 x i64> @test_gather_16i64(<16 x i64*> %ptrs, <16 x i1> %mask, <16 x i64> %src0) { +; KNL_64-LABEL: test_gather_16i64: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpmovsxbd %xmm2, %zmm2 +; KNL_64-NEXT: vpslld $31, %zmm2, %zmm2 +; KNL_64-NEXT: vptestmd %zmm2, %zmm2, %k1 +; KNL_64-NEXT: kshiftrw $8, %k1, %k2 +; KNL_64-NEXT: vpgatherqq (,%zmm0), %zmm3 {%k1} +; KNL_64-NEXT: vpgatherqq (,%zmm1), %zmm4 {%k2} +; KNL_64-NEXT: vmovaps %zmm3, %zmm0 +; KNL_64-NEXT: vmovaps %zmm4, %zmm1 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test_gather_16i64: +; KNL_32: # BB#0: +; KNL_32-NEXT: pushl %ebp +; KNL_32-NEXT: .Ltmp0: +; KNL_32-NEXT: .cfi_def_cfa_offset 8 +; KNL_32-NEXT: .Ltmp1: +; KNL_32-NEXT: .cfi_offset %ebp, -8 +; KNL_32-NEXT: movl %esp, %ebp +; KNL_32-NEXT: .Ltmp2: +; KNL_32-NEXT: .cfi_def_cfa_register %ebp +; KNL_32-NEXT: andl $-64, %esp +; KNL_32-NEXT: subl $64, %esp +; KNL_32-NEXT: vpmovsxbd %xmm1, %zmm1 +; KNL_32-NEXT: vpslld $31, %zmm1, %zmm1 +; KNL_32-NEXT: vptestmd %zmm1, %zmm1, %k1 +; KNL_32-NEXT: vmovdqa64 8(%ebp), %zmm1 +; KNL_32-NEXT: kshiftrw $8, %k1, %k2 +; KNL_32-NEXT: vpgatherdq (,%ymm0), %zmm2 {%k1} +; KNL_32-NEXT: vextracti64x4 $1, %zmm0, %ymm0 +; KNL_32-NEXT: vpgatherdq (,%ymm0), %zmm1 {%k2} +; KNL_32-NEXT: vmovaps %zmm2, %zmm0 +; KNL_32-NEXT: movl %ebp, %esp +; KNL_32-NEXT: popl %ebp +; KNL_32-NEXT: retl +; +; SKX-LABEL: test_gather_16i64: +; SKX: # BB#0: +; SKX-NEXT: vpmovsxbd %xmm2, %zmm2 +; SKX-NEXT: vpslld $31, %zmm2, %zmm2 +; SKX-NEXT: vptestmd %zmm2, %zmm2, %k1 +; SKX-NEXT: kshiftrw $8, %k1, %k2 +; SKX-NEXT: vpgatherqq (,%zmm0), %zmm3 {%k1} +; SKX-NEXT: vpgatherqq (,%zmm1), %zmm4 {%k2} +; SKX-NEXT: vmovaps %zmm3, %zmm0 +; SKX-NEXT: vmovaps %zmm4, %zmm1 +; SKX-NEXT: retq +; +; SKX_32-LABEL: test_gather_16i64: +; SKX_32: # BB#0: +; SKX_32-NEXT: pushl %ebp +; SKX_32-NEXT: .Ltmp1: +; SKX_32-NEXT: .cfi_def_cfa_offset 8 +; SKX_32-NEXT: .Ltmp2: +; SKX_32-NEXT: .cfi_offset %ebp, -8 +; SKX_32-NEXT: movl %esp, %ebp +; SKX_32-NEXT: .Ltmp3: +; SKX_32-NEXT: .cfi_def_cfa_register %ebp +; SKX_32-NEXT: andl $-64, %esp +; SKX_32-NEXT: subl $64, %esp +; SKX_32-NEXT: vpmovsxbd %xmm1, %zmm1 +; SKX_32-NEXT: vpslld $31, %zmm1, %zmm1 +; SKX_32-NEXT: vptestmd %zmm1, %zmm1, %k1 +; SKX_32-NEXT: vmovdqa64 8(%ebp), %zmm1 +; SKX_32-NEXT: kshiftrw $8, %k1, %k2 +; SKX_32-NEXT: vpgatherdq (,%ymm0), %zmm2 {%k1} +; SKX_32-NEXT: vextracti32x8 $1, %zmm0, %ymm0 +; SKX_32-NEXT: vpgatherdq (,%ymm0), %zmm1 {%k2} +; SKX_32-NEXT: vmovaps %zmm2, %zmm0 +; SKX_32-NEXT: movl %ebp, %esp +; SKX_32-NEXT: popl %ebp +; SKX_32-NEXT: retl + %res = call <16 x i64> @llvm.masked.gather.v16i64(<16 x i64*> %ptrs, i32 4, <16 x i1> %mask, <16 x i64> %src0) + ret <16 x i64> %res +} +declare <16 x i64> @llvm.masked.gather.v16i64(<16 x i64*> %ptrs, i32, <16 x i1> %mask, <16 x i64> %src0) +define <16 x float> @test_gather_16f32(<16 x float*> %ptrs, <16 x i1> %mask, <16 x float> %src0) { +; KNL_64-LABEL: test_gather_16f32: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpmovsxbd %xmm2, %zmm2 +; KNL_64-NEXT: vpslld $31, %zmm2, %zmm2 +; KNL_64-NEXT: vptestmd %zmm2, %zmm2, %k1 +; KNL_64-NEXT: vextractf64x4 $1, %zmm3, %ymm2 +; KNL_64-NEXT: kshiftrw $8, %k1, %k2 +; KNL_64-NEXT: vgatherqps (,%zmm1), %ymm2 {%k2} +; KNL_64-NEXT: vgatherqps (,%zmm0), %ymm3 {%k1} +; KNL_64-NEXT: vinsertf64x4 $1, %ymm2, %zmm3, %zmm0 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test_gather_16f32: +; KNL_32: # BB#0: +; KNL_32-NEXT: vpmovsxbd %xmm1, %zmm1 +; KNL_32-NEXT: vpslld $31, %zmm1, %zmm1 +; KNL_32-NEXT: vptestmd %zmm1, %zmm1, %k1 +; KNL_32-NEXT: vgatherdps (,%zmm0), %zmm2 {%k1} +; KNL_32-NEXT: vmovaps %zmm2, %zmm0 +; KNL_32-NEXT: retl +; +; SKX-LABEL: test_gather_16f32: +; SKX: # BB#0: +; SKX-NEXT: vpmovsxbd %xmm2, %zmm2 +; SKX-NEXT: vpslld $31, %zmm2, %zmm2 +; SKX-NEXT: vptestmd %zmm2, %zmm2, %k1 +; SKX-NEXT: vextractf32x8 $1, %zmm3, %ymm2 +; SKX-NEXT: kshiftrw $8, %k1, %k2 +; SKX-NEXT: vgatherqps (,%zmm1), %ymm2 {%k2} +; SKX-NEXT: vgatherqps (,%zmm0), %ymm3 {%k1} +; SKX-NEXT: vinsertf32x8 $1, %ymm2, %zmm3, %zmm0 +; SKX-NEXT: retq +; +; SKX_32-LABEL: test_gather_16f32: +; SKX_32: # BB#0: +; SKX_32-NEXT: vpmovsxbd %xmm1, %zmm1 +; SKX_32-NEXT: vpslld $31, %zmm1, %zmm1 +; SKX_32-NEXT: vptestmd %zmm1, %zmm1, %k1 +; SKX_32-NEXT: vgatherdps (,%zmm0), %zmm2 {%k1} +; SKX_32-NEXT: vmovaps %zmm2, %zmm0 +; SKX_32-NEXT: retl + %res = call <16 x float> @llvm.masked.gather.v16f32(<16 x float*> %ptrs, i32 4, <16 x i1> %mask, <16 x float> %src0) + ret <16 x float> %res +} +define <16 x double> @test_gather_16f64(<16 x double*> %ptrs, <16 x i1> %mask, <16 x double> %src0) { +; KNL_64-LABEL: test_gather_16f64: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpmovsxbd %xmm2, %zmm2 +; KNL_64-NEXT: vpslld $31, %zmm2, %zmm2 +; KNL_64-NEXT: vptestmd %zmm2, %zmm2, %k1 +; KNL_64-NEXT: kshiftrw $8, %k1, %k2 +; KNL_64-NEXT: vgatherqpd (,%zmm0), %zmm3 {%k1} +; KNL_64-NEXT: vgatherqpd (,%zmm1), %zmm4 {%k2} +; KNL_64-NEXT: vmovaps %zmm3, %zmm0 +; KNL_64-NEXT: vmovaps %zmm4, %zmm1 +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test_gather_16f64: +; KNL_32: # BB#0: +; KNL_32-NEXT: pushl %ebp +; KNL_32-NEXT: .Ltmp3: +; KNL_32-NEXT: .cfi_def_cfa_offset 8 +; KNL_32-NEXT: .Ltmp4: +; KNL_32-NEXT: .cfi_offset %ebp, -8 +; KNL_32-NEXT: movl %esp, %ebp +; KNL_32-NEXT: .Ltmp5: +; KNL_32-NEXT: .cfi_def_cfa_register %ebp +; KNL_32-NEXT: andl $-64, %esp +; KNL_32-NEXT: subl $64, %esp +; KNL_32-NEXT: vpmovsxbd %xmm1, %zmm1 +; KNL_32-NEXT: vpslld $31, %zmm1, %zmm1 +; KNL_32-NEXT: vptestmd %zmm1, %zmm1, %k1 +; KNL_32-NEXT: vmovapd 8(%ebp), %zmm1 +; KNL_32-NEXT: kshiftrw $8, %k1, %k2 +; KNL_32-NEXT: vgatherdpd (,%ymm0), %zmm2 {%k1} +; KNL_32-NEXT: vextracti64x4 $1, %zmm0, %ymm0 +; KNL_32-NEXT: vgatherdpd (,%ymm0), %zmm1 {%k2} +; KNL_32-NEXT: vmovaps %zmm2, %zmm0 +; KNL_32-NEXT: movl %ebp, %esp +; KNL_32-NEXT: popl %ebp +; KNL_32-NEXT: retl +; +; SKX-LABEL: test_gather_16f64: +; SKX: # BB#0: +; SKX-NEXT: vpmovsxbd %xmm2, %zmm2 +; SKX-NEXT: vpslld $31, %zmm2, %zmm2 +; SKX-NEXT: vptestmd %zmm2, %zmm2, %k1 +; SKX-NEXT: kshiftrw $8, %k1, %k2 +; SKX-NEXT: vgatherqpd (,%zmm0), %zmm3 {%k1} +; SKX-NEXT: vgatherqpd (,%zmm1), %zmm4 {%k2} +; SKX-NEXT: vmovaps %zmm3, %zmm0 +; SKX-NEXT: vmovaps %zmm4, %zmm1 +; SKX-NEXT: retq +; +; SKX_32-LABEL: test_gather_16f64: +; SKX_32: # BB#0: +; SKX_32-NEXT: pushl %ebp +; SKX_32-NEXT: .Ltmp4: +; SKX_32-NEXT: .cfi_def_cfa_offset 8 +; SKX_32-NEXT: .Ltmp5: +; SKX_32-NEXT: .cfi_offset %ebp, -8 +; SKX_32-NEXT: movl %esp, %ebp +; SKX_32-NEXT: .Ltmp6: +; SKX_32-NEXT: .cfi_def_cfa_register %ebp +; SKX_32-NEXT: andl $-64, %esp +; SKX_32-NEXT: subl $64, %esp +; SKX_32-NEXT: vpmovsxbd %xmm1, %zmm1 +; SKX_32-NEXT: vpslld $31, %zmm1, %zmm1 +; SKX_32-NEXT: vptestmd %zmm1, %zmm1, %k1 +; SKX_32-NEXT: vmovapd 8(%ebp), %zmm1 +; SKX_32-NEXT: kshiftrw $8, %k1, %k2 +; SKX_32-NEXT: vgatherdpd (,%ymm0), %zmm2 {%k1} +; SKX_32-NEXT: vextracti32x8 $1, %zmm0, %ymm0 +; SKX_32-NEXT: vgatherdpd (,%ymm0), %zmm1 {%k2} +; SKX_32-NEXT: vmovaps %zmm2, %zmm0 +; SKX_32-NEXT: movl %ebp, %esp +; SKX_32-NEXT: popl %ebp +; SKX_32-NEXT: retl + %res = call <16 x double> @llvm.masked.gather.v16f64(<16 x double*> %ptrs, i32 4, <16 x i1> %mask, <16 x double> %src0) + ret <16 x double> %res +} +declare <16 x double> @llvm.masked.gather.v16f64(<16 x double*> %ptrs, i32, <16 x i1> %mask, <16 x double> %src0) +define void @test_scatter_16i32(<16 x i32*> %ptrs, <16 x i1> %mask, <16 x i32> %src0) { +; KNL_64-LABEL: test_scatter_16i32: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpmovsxbd %xmm2, %zmm2 +; KNL_64-NEXT: vpslld $31, %zmm2, %zmm2 +; KNL_64-NEXT: vptestmd %zmm2, %zmm2, %k1 +; KNL_64-NEXT: kshiftrw $8, %k1, %k2 +; KNL_64-NEXT: vpscatterqd %ymm3, (,%zmm0) {%k1} +; KNL_64-NEXT: vextracti64x4 $1, %zmm3, %ymm0 +; KNL_64-NEXT: vpscatterqd %ymm0, (,%zmm1) {%k2} +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test_scatter_16i32: +; KNL_32: # BB#0: +; KNL_32-NEXT: vpmovsxbd %xmm1, %zmm1 +; KNL_32-NEXT: vpslld $31, %zmm1, %zmm1 +; KNL_32-NEXT: vptestmd %zmm1, %zmm1, %k1 +; KNL_32-NEXT: vpscatterdd %zmm2, (,%zmm0) {%k1} +; KNL_32-NEXT: retl +; +; SKX-LABEL: test_scatter_16i32: +; SKX: # BB#0: +; SKX-NEXT: vpmovsxbd %xmm2, %zmm2 +; SKX-NEXT: vpslld $31, %zmm2, %zmm2 +; SKX-NEXT: vptestmd %zmm2, %zmm2, %k1 +; SKX-NEXT: kshiftrw $8, %k1, %k2 +; SKX-NEXT: vpscatterqd %ymm3, (,%zmm0) {%k1} +; SKX-NEXT: vextracti32x8 $1, %zmm3, %ymm0 +; SKX-NEXT: vpscatterqd %ymm0, (,%zmm1) {%k2} +; SKX-NEXT: retq +; +; SKX_32-LABEL: test_scatter_16i32: +; SKX_32: # BB#0: +; SKX_32-NEXT: vpmovsxbd %xmm1, %zmm1 +; SKX_32-NEXT: vpslld $31, %zmm1, %zmm1 +; SKX_32-NEXT: vptestmd %zmm1, %zmm1, %k1 +; SKX_32-NEXT: vpscatterdd %zmm2, (,%zmm0) {%k1} +; SKX_32-NEXT: retl + call void @llvm.masked.scatter.v16i32(<16 x i32> %src0, <16 x i32*> %ptrs, i32 4, <16 x i1> %mask) + ret void +} +define void @test_scatter_16i64(<16 x i64*> %ptrs, <16 x i1> %mask, <16 x i64> %src0) { +; KNL_64-LABEL: test_scatter_16i64: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpmovsxbd %xmm2, %zmm2 +; KNL_64-NEXT: vpslld $31, %zmm2, %zmm2 +; KNL_64-NEXT: vptestmd %zmm2, %zmm2, %k1 +; KNL_64-NEXT: kshiftrw $8, %k1, %k2 +; KNL_64-NEXT: vpscatterqq %zmm3, (,%zmm0) {%k1} +; KNL_64-NEXT: vpscatterqq %zmm4, (,%zmm1) {%k2} +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test_scatter_16i64: +; KNL_32: # BB#0: +; KNL_32-NEXT: pushl %ebp +; KNL_32-NEXT: .Ltmp6: +; KNL_32-NEXT: .cfi_def_cfa_offset 8 +; KNL_32-NEXT: .Ltmp7: +; KNL_32-NEXT: .cfi_offset %ebp, -8 +; KNL_32-NEXT: movl %esp, %ebp +; KNL_32-NEXT: .Ltmp8: +; KNL_32-NEXT: .cfi_def_cfa_register %ebp +; KNL_32-NEXT: andl $-64, %esp +; KNL_32-NEXT: subl $64, %esp +; KNL_32-NEXT: vpmovsxbd %xmm1, %zmm1 +; KNL_32-NEXT: vpslld $31, %zmm1, %zmm1 +; KNL_32-NEXT: vptestmd %zmm1, %zmm1, %k1 +; KNL_32-NEXT: vmovdqa64 8(%ebp), %zmm1 +; KNL_32-NEXT: kshiftrw $8, %k1, %k2 +; KNL_32-NEXT: vpscatterdq %zmm2, (,%ymm0) {%k1} +; KNL_32-NEXT: vextracti64x4 $1, %zmm0, %ymm0 +; KNL_32-NEXT: vpscatterdq %zmm1, (,%ymm0) {%k2} +; KNL_32-NEXT: movl %ebp, %esp +; KNL_32-NEXT: popl %ebp +; KNL_32-NEXT: retl +; +; SKX-LABEL: test_scatter_16i64: +; SKX: # BB#0: +; SKX-NEXT: vpmovsxbd %xmm2, %zmm2 +; SKX-NEXT: vpslld $31, %zmm2, %zmm2 +; SKX-NEXT: vptestmd %zmm2, %zmm2, %k1 +; SKX-NEXT: kshiftrw $8, %k1, %k2 +; SKX-NEXT: vpscatterqq %zmm3, (,%zmm0) {%k1} +; SKX-NEXT: vpscatterqq %zmm4, (,%zmm1) {%k2} +; SKX-NEXT: retq +; +; SKX_32-LABEL: test_scatter_16i64: +; SKX_32: # BB#0: +; SKX_32-NEXT: pushl %ebp +; SKX_32-NEXT: .Ltmp7: +; SKX_32-NEXT: .cfi_def_cfa_offset 8 +; SKX_32-NEXT: .Ltmp8: +; SKX_32-NEXT: .cfi_offset %ebp, -8 +; SKX_32-NEXT: movl %esp, %ebp +; SKX_32-NEXT: .Ltmp9: +; SKX_32-NEXT: .cfi_def_cfa_register %ebp +; SKX_32-NEXT: andl $-64, %esp +; SKX_32-NEXT: subl $64, %esp +; SKX_32-NEXT: vpmovsxbd %xmm1, %zmm1 +; SKX_32-NEXT: vpslld $31, %zmm1, %zmm1 +; SKX_32-NEXT: vptestmd %zmm1, %zmm1, %k1 +; SKX_32-NEXT: vmovdqa64 8(%ebp), %zmm1 +; SKX_32-NEXT: kshiftrw $8, %k1, %k2 +; SKX_32-NEXT: vpscatterdq %zmm2, (,%ymm0) {%k1} +; SKX_32-NEXT: vextracti32x8 $1, %zmm0, %ymm0 +; SKX_32-NEXT: vpscatterdq %zmm1, (,%ymm0) {%k2} +; SKX_32-NEXT: movl %ebp, %esp +; SKX_32-NEXT: popl %ebp +; SKX_32-NEXT: retl + call void @llvm.masked.scatter.v16i64(<16 x i64> %src0, <16 x i64*> %ptrs, i32 4, <16 x i1> %mask) + ret void +} +declare void @llvm.masked.scatter.v16i64(<16 x i64> %src0, <16 x i64*> %ptrs, i32, <16 x i1> %mask) +define void @test_scatter_16f32(<16 x float*> %ptrs, <16 x i1> %mask, <16 x float> %src0) { +; KNL_64-LABEL: test_scatter_16f32: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpmovsxbd %xmm2, %zmm2 +; KNL_64-NEXT: vpslld $31, %zmm2, %zmm2 +; KNL_64-NEXT: vptestmd %zmm2, %zmm2, %k1 +; KNL_64-NEXT: kshiftrw $8, %k1, %k2 +; KNL_64-NEXT: vscatterqps %ymm3, (,%zmm0) {%k1} +; KNL_64-NEXT: vextractf64x4 $1, %zmm3, %ymm0 +; KNL_64-NEXT: vscatterqps %ymm0, (,%zmm1) {%k2} +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test_scatter_16f32: +; KNL_32: # BB#0: +; KNL_32-NEXT: vpmovsxbd %xmm1, %zmm1 +; KNL_32-NEXT: vpslld $31, %zmm1, %zmm1 +; KNL_32-NEXT: vptestmd %zmm1, %zmm1, %k1 +; KNL_32-NEXT: vscatterdps %zmm2, (,%zmm0) {%k1} +; KNL_32-NEXT: retl +; +; SKX-LABEL: test_scatter_16f32: +; SKX: # BB#0: +; SKX-NEXT: vpmovsxbd %xmm2, %zmm2 +; SKX-NEXT: vpslld $31, %zmm2, %zmm2 +; SKX-NEXT: vptestmd %zmm2, %zmm2, %k1 +; SKX-NEXT: kshiftrw $8, %k1, %k2 +; SKX-NEXT: vscatterqps %ymm3, (,%zmm0) {%k1} +; SKX-NEXT: vextractf32x8 $1, %zmm3, %ymm0 +; SKX-NEXT: vscatterqps %ymm0, (,%zmm1) {%k2} +; SKX-NEXT: retq +; +; SKX_32-LABEL: test_scatter_16f32: +; SKX_32: # BB#0: +; SKX_32-NEXT: vpmovsxbd %xmm1, %zmm1 +; SKX_32-NEXT: vpslld $31, %zmm1, %zmm1 +; SKX_32-NEXT: vptestmd %zmm1, %zmm1, %k1 +; SKX_32-NEXT: vscatterdps %zmm2, (,%zmm0) {%k1} +; SKX_32-NEXT: retl + call void @llvm.masked.scatter.v16f32(<16 x float> %src0, <16 x float*> %ptrs, i32 4, <16 x i1> %mask) + ret void +} +declare void @llvm.masked.scatter.v16f32(<16 x float> %src0, <16 x float*> %ptrs, i32, <16 x i1> %mask) +define void @test_scatter_16f64(<16 x double*> %ptrs, <16 x i1> %mask, <16 x double> %src0) { +; KNL_64-LABEL: test_scatter_16f64: +; KNL_64: # BB#0: +; KNL_64-NEXT: vpmovsxbd %xmm2, %zmm2 +; KNL_64-NEXT: vpslld $31, %zmm2, %zmm2 +; KNL_64-NEXT: vptestmd %zmm2, %zmm2, %k1 +; KNL_64-NEXT: kshiftrw $8, %k1, %k2 +; KNL_64-NEXT: vscatterqpd %zmm3, (,%zmm0) {%k1} +; KNL_64-NEXT: vscatterqpd %zmm4, (,%zmm1) {%k2} +; KNL_64-NEXT: retq +; +; KNL_32-LABEL: test_scatter_16f64: +; KNL_32: # BB#0: +; KNL_32-NEXT: pushl %ebp +; KNL_32-NEXT: .Ltmp9: +; KNL_32-NEXT: .cfi_def_cfa_offset 8 +; KNL_32-NEXT: .Ltmp10: +; KNL_32-NEXT: .cfi_offset %ebp, -8 +; KNL_32-NEXT: movl %esp, %ebp +; KNL_32-NEXT: .Ltmp11: +; KNL_32-NEXT: .cfi_def_cfa_register %ebp +; KNL_32-NEXT: andl $-64, %esp +; KNL_32-NEXT: subl $64, %esp +; KNL_32-NEXT: vpmovsxbd %xmm1, %zmm1 +; KNL_32-NEXT: vpslld $31, %zmm1, %zmm1 +; KNL_32-NEXT: vptestmd %zmm1, %zmm1, %k1 +; KNL_32-NEXT: vmovapd 8(%ebp), %zmm1 +; KNL_32-NEXT: kshiftrw $8, %k1, %k2 +; KNL_32-NEXT: vscatterdpd %zmm2, (,%ymm0) {%k1} +; KNL_32-NEXT: vextracti64x4 $1, %zmm0, %ymm0 +; KNL_32-NEXT: vscatterdpd %zmm1, (,%ymm0) {%k2} +; KNL_32-NEXT: movl %ebp, %esp +; KNL_32-NEXT: popl %ebp +; KNL_32-NEXT: retl +; +; SKX-LABEL: test_scatter_16f64: +; SKX: # BB#0: +; SKX-NEXT: vpmovsxbd %xmm2, %zmm2 +; SKX-NEXT: vpslld $31, %zmm2, %zmm2 +; SKX-NEXT: vptestmd %zmm2, %zmm2, %k1 +; SKX-NEXT: kshiftrw $8, %k1, %k2 +; SKX-NEXT: vscatterqpd %zmm3, (,%zmm0) {%k1} +; SKX-NEXT: vscatterqpd %zmm4, (,%zmm1) {%k2} +; SKX-NEXT: retq +; +; SKX_32-LABEL: test_scatter_16f64: +; SKX_32: # BB#0: +; SKX_32-NEXT: pushl %ebp +; SKX_32-NEXT: .Ltmp10: +; SKX_32-NEXT: .cfi_def_cfa_offset 8 +; SKX_32-NEXT: .Ltmp11: +; SKX_32-NEXT: .cfi_offset %ebp, -8 +; SKX_32-NEXT: movl %esp, %ebp +; SKX_32-NEXT: .Ltmp12: +; SKX_32-NEXT: .cfi_def_cfa_register %ebp +; SKX_32-NEXT: andl $-64, %esp +; SKX_32-NEXT: subl $64, %esp +; SKX_32-NEXT: vpmovsxbd %xmm1, %zmm1 +; SKX_32-NEXT: vpslld $31, %zmm1, %zmm1 +; SKX_32-NEXT: vptestmd %zmm1, %zmm1, %k1 +; SKX_32-NEXT: vmovapd 8(%ebp), %zmm1 +; SKX_32-NEXT: kshiftrw $8, %k1, %k2 +; SKX_32-NEXT: vscatterdpd %zmm2, (,%ymm0) {%k1} +; SKX_32-NEXT: vextracti32x8 $1, %zmm0, %ymm0 +; SKX_32-NEXT: vscatterdpd %zmm1, (,%ymm0) {%k2} +; SKX_32-NEXT: movl %ebp, %esp +; SKX_32-NEXT: popl %ebp +; SKX_32-NEXT: retl + call void @llvm.masked.scatter.v16f64(<16 x double> %src0, <16 x double*> %ptrs, i32 4, <16 x i1> %mask) + ret void +} +declare void @llvm.masked.scatter.v16f64(<16 x double> %src0, <16 x double*> %ptrs, i32, <16 x i1> %mask) diff --git a/test/CodeGen/X86/masked_memop.ll b/test/CodeGen/X86/masked_memop.ll index 6c16e634a59f..c29933e266b2 100644 --- a/test/CodeGen/X86/masked_memop.ll +++ b/test/CodeGen/X86/masked_memop.ll @@ -1,7 +1,8 @@ -; RUN: llc -mtriple=x86_64-apple-darwin -mcpu=knl < %s | FileCheck %s -check-prefix=AVX512 -; RUN: llc -mtriple=x86_64-apple-darwin -mcpu=core-avx2 < %s | FileCheck %s -check-prefix=AVX2 -; RUN: opt -mtriple=x86_64-apple-darwin -codegenprepare -mcpu=corei7-avx -S < %s | FileCheck %s -check-prefix=AVX_SCALAR -; RUN: llc -mtriple=x86_64-apple-darwin -mcpu=skx < %s | FileCheck %s -check-prefix=SKX +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=x86_64-apple-darwin -mcpu=knl < %s | FileCheck %s --check-prefix=AVX512 +; RUN: llc -mtriple=x86_64-apple-darwin -mcpu=core-avx2 < %s | FileCheck %s --check-prefix=AVX2 +; RUN: opt -mtriple=x86_64-apple-darwin -codegenprepare -mcpu=corei7-avx -S < %s | FileCheck %s --check-prefix=AVX_SCALAR +; RUN: llc -mtriple=x86_64-apple-darwin -mcpu=skx < %s | FileCheck %s --check-prefix=SKX ; AVX512-LABEL: test1 ; AVX512: vmovdqu32 (%rdi), %zmm0 {%k1} {z} @@ -139,18 +140,55 @@ define <4 x double> @test10(<4 x i32> %trigger, <4 x double>* %addr, <4 x double ret <4 x double> %res } -; AVX2-LABEL: test11 +; AVX2-LABEL: test11a ; AVX2: vmaskmovps ; AVX2: vblendvps -; SKX-LABEL: test11 -; SKX: vmovaps {{.*}}{%k1} -define <8 x float> @test11(<8 x i32> %trigger, <8 x float>* %addr, <8 x float> %dst) { +; SKX-LABEL: test11a +; SKX: vmovaps (%rdi), %ymm1 {%k1} +; AVX512-LABEL: test11a +; AVX512: kshiftlw $8 +; AVX512: kshiftrw $8 +; AVX512: vmovups (%rdi), %zmm1 {%k1} +define <8 x float> @test11a(<8 x i32> %trigger, <8 x float>* %addr, <8 x float> %dst) { %mask = icmp eq <8 x i32> %trigger, zeroinitializer %res = call <8 x float> @llvm.masked.load.v8f32(<8 x float>* %addr, i32 32, <8 x i1>%mask, <8 x float>%dst) ret <8 x float> %res } +; SKX-LABEL: test11b +; SKX: vmovdqu32 (%rdi), %ymm1 {%k1} +; AVX512-LABEL: test11b +; AVX512: kshiftlw $8 +; AVX512: kshiftrw $8 +; AVX512: vmovdqu32 (%rdi), %zmm1 {%k1} +define <8 x i32> @test11b(<8 x i1> %mask, <8 x i32>* %addr, <8 x i32> %dst) { + %res = call <8 x i32> @llvm.masked.load.v8i32(<8 x i32>* %addr, i32 4, <8 x i1>%mask, <8 x i32>%dst) + ret <8 x i32> %res +} + +; SKX-LABEL: test11c +; SKX: vmovaps (%rdi), %ymm0 {%k1} {z} +; AVX512-LABEL: test11c +; AVX512: kshiftlw $8 +; AVX512: kshiftrw $8 +; AVX512: vmovups (%rdi), %zmm0 {%k1} {z} +define <8 x float> @test11c(<8 x i1> %mask, <8 x float>* %addr) { + %res = call <8 x float> @llvm.masked.load.v8f32(<8 x float>* %addr, i32 32, <8 x i1> %mask, <8 x float> zeroinitializer) + ret <8 x float> %res +} + +; SKX-LABEL: test11d +; SKX: vmovdqu32 (%rdi), %ymm0 {%k1} {z} +; AVX512-LABEL: test11d +; AVX512: kshiftlw $8 +; AVX512: kshiftrw $8 +; AVX512: vmovdqu32 (%rdi), %zmm0 {%k1} {z} +define <8 x i32> @test11d(<8 x i1> %mask, <8 x i32>* %addr) { + %res = call <8 x i32> @llvm.masked.load.v8i32(<8 x i32>* %addr, i32 4, <8 x i1> %mask, <8 x i32> zeroinitializer) + ret <8 x i32> %res +} + ; AVX2-LABEL: test12 ; AVX2: vpmaskmovd %ymm @@ -190,10 +228,13 @@ define void @test14(<2 x i32> %trigger, <2 x float>* %addr, <2 x float> %val) { ; AVX2-LABEL: test15 ; AVX2: vpmaskmovd -; SKX-LABEL: test15 -; SKX: kshiftl -; SKX: kshiftr -; SKX: vmovdqu32 {{.*}}{%k1} +; SKX-LABEL: test15: +; SKX: ## BB#0: +; SKX-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; SKX-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3] +; SKX-NEXT: vpcmpeqq %xmm2, %xmm0, %k1 +; SKX-NEXT: vpmovqd %xmm1, (%rdi) {%k1} +; SKX-NEXT: retq define void @test15(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %val) { %mask = icmp eq <2 x i32> %trigger, zeroinitializer call void @llvm.masked.store.v2i32(<2 x i32>%val, <2 x i32>* %addr, i32 4, <2 x i1>%mask) @@ -232,12 +273,58 @@ define <2 x i32> @test17(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %dst) { ; AVX2-LABEL: test18 ; AVX2: vmaskmovps ; AVX2-NOT: blend +; AVX2: ret define <2 x float> @test18(<2 x i32> %trigger, <2 x float>* %addr) { +; SKX-LABEL: test18: +; SKX: ## BB#0: +; SKX-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; SKX-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] +; SKX-NEXT: vpcmpeqq %xmm1, %xmm0, %k0 +; SKX-NEXT: kshiftlw $2, %k0, %k0 +; SKX-NEXT: kshiftrw $2, %k0, %k1 +; SKX-NEXT: vmovups (%rdi), %xmm0 {%k1} {z} +; SKX-NEXT: retq %mask = icmp eq <2 x i32> %trigger, zeroinitializer %res = call <2 x float> @llvm.masked.load.v2f32(<2 x float>* %addr, i32 4, <2 x i1>%mask, <2 x float>undef) ret <2 x float> %res } +; AVX_SCALAR-LABEL: test19 +; AVX_SCALAR: load <4 x float>, <4 x float>* %addr, align 4 + +define <4 x float> @test19(<4 x i32> %trigger, <4 x float>* %addr) { + %mask = icmp eq <4 x i32> %trigger, zeroinitializer + %res = call <4 x float> @llvm.masked.load.v4f32(<4 x float>* %addr, i32 4, <4 x i1>, <4 x float>undef) + ret <4 x float> %res +} + +; AVX_SCALAR-LABEL: test20 +; AVX_SCALAR: load float, {{.*}}, align 4 +; AVX_SCALAR: insertelement <4 x float> undef, float +; AVX_SCALAR: select <4 x i1> + +define <4 x float> @test20(<4 x i32> %trigger, <4 x float>* %addr, <4 x float> %src0) { + %mask = icmp eq <4 x i32> %trigger, zeroinitializer + %res = call <4 x float> @llvm.masked.load.v4f32(<4 x float>* %addr, i32 16, <4 x i1>, <4 x float> %src0) + ret <4 x float> %res +} + +; AVX_SCALAR-LABEL: test21 +; AVX_SCALAR: store <4 x i32> %val +define void @test21(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %val) { + %mask = icmp eq <4 x i32> %trigger, zeroinitializer + call void @llvm.masked.store.v4i32(<4 x i32>%val, <4 x i32>* %addr, i32 4, <4 x i1>) + ret void +} + +; AVX_SCALAR-LABEL: test22 +; AVX_SCALAR: extractelement <4 x i32> %val, i32 0 +; AVX_SCALAR: store i32 +define void @test22(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %val) { + %mask = icmp eq <4 x i32> %trigger, zeroinitializer + call void @llvm.masked.store.v4i32(<4 x i32>%val, <4 x i32>* %addr, i32 4, <4 x i1>) + ret void +} declare <16 x i32> @llvm.masked.load.v16i32(<16 x i32>*, i32, <16 x i1>, <16 x i32>) declare <4 x i32> @llvm.masked.load.v4i32(<4 x i32>*, i32, <4 x i1>, <4 x i32>) @@ -251,6 +338,7 @@ declare void @llvm.masked.store.v16f32(<16 x float>, <16 x float>*, i32, <16 x i declare void @llvm.masked.store.v16f32p(<16 x float>*, <16 x float>**, i32, <16 x i1>) declare <16 x float> @llvm.masked.load.v16f32(<16 x float>*, i32, <16 x i1>, <16 x float>) declare <8 x float> @llvm.masked.load.v8f32(<8 x float>*, i32, <8 x i1>, <8 x float>) +declare <8 x i32> @llvm.masked.load.v8i32(<8 x i32>*, i32, <8 x i1>, <8 x i32>) declare <4 x float> @llvm.masked.load.v4f32(<4 x float>*, i32, <4 x i1>, <4 x float>) declare <2 x float> @llvm.masked.load.v2f32(<2 x float>*, i32, <2 x i1>, <2 x float>) declare <8 x double> @llvm.masked.load.v8f64(<8 x double>*, i32, <8 x i1>, <8 x double>) @@ -260,3 +348,415 @@ declare void @llvm.masked.store.v8f64(<8 x double>, <8 x double>*, i32, <8 x i1> declare void @llvm.masked.store.v2f64(<2 x double>, <2 x double>*, i32, <2 x i1>) declare void @llvm.masked.store.v2i64(<2 x i64>, <2 x i64>*, i32, <2 x i1>) +declare <16 x i32*> @llvm.masked.load.v16p0i32(<16 x i32*>*, i32, <16 x i1>, <16 x i32*>) + +; AVX512-LABEL: test23 +; AVX512: vmovdqu64 64(%rdi), %zmm1 {%k2} {z} +; AVX512: vmovdqu64 (%rdi), %zmm0 {%k1} {z} + +define <16 x i32*> @test23(<16 x i32*> %trigger, <16 x i32*>* %addr) { + %mask = icmp eq <16 x i32*> %trigger, zeroinitializer + %res = call <16 x i32*> @llvm.masked.load.v16p0i32(<16 x i32*>* %addr, i32 4, <16 x i1>%mask, <16 x i32*>zeroinitializer) + ret <16 x i32*> %res +} + +%mystruct = type { i16, i16, [1 x i8*] } + +declare <16 x %mystruct*> @llvm.masked.load.v16p0mystruct(<16 x %mystruct*>*, i32, <16 x i1>, <16 x %mystruct*>) + +define <16 x %mystruct*> @test24(<16 x i1> %mask, <16 x %mystruct*>* %addr) { +; AVX512-LABEL: test24: +; AVX512: ## BB#0: +; AVX512-NEXT: vpmovsxbd %xmm0, %zmm0 +; AVX512-NEXT: vpslld $31, %zmm0, %zmm0 +; AVX512-NEXT: vptestmd %zmm0, %zmm0, %k1 +; AVX512-NEXT: vmovdqu64 (%rdi), %zmm0 {%k1} {z} +; AVX512-NEXT: kshiftrw $8, %k1, %k1 +; AVX512-NEXT: vmovdqu64 64(%rdi), %zmm1 {%k1} {z} +; AVX512-NEXT: retq +; +; AVX2-LABEL: test24: +; AVX2: ## BB#0: +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm1, %xmm1 +; AVX2-NEXT: vpsrad $31, %xmm1, %xmm1 +; AVX2-NEXT: vpmovsxdq %xmm1, %ymm1 +; AVX2-NEXT: vpmaskmovq (%rdi), %ymm1, %ymm4 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[3,1,2,3] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm1, %xmm1 +; AVX2-NEXT: vpsrad $31, %xmm1, %xmm1 +; AVX2-NEXT: vpmovsxdq %xmm1, %ymm1 +; AVX2-NEXT: vpmaskmovq 96(%rdi), %ymm1, %ymm3 +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm1, %xmm1 +; AVX2-NEXT: vpsrad $31, %xmm1, %xmm1 +; AVX2-NEXT: vpmovsxdq %xmm1, %ymm1 +; AVX2-NEXT: vpmaskmovq 64(%rdi), %ymm1, %ymm2 +; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm0, %xmm0 +; AVX2-NEXT: vpsrad $31, %xmm0, %xmm0 +; AVX2-NEXT: vpmovsxdq %xmm0, %ymm0 +; AVX2-NEXT: vpmaskmovq 32(%rdi), %ymm0, %ymm1 +; AVX2-NEXT: vmovdqa %ymm4, %ymm0 +; AVX2-NEXT: retq +; +; SKX-LABEL: test24: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm0, %xmm0 +; SKX-NEXT: vpmovb2m %xmm0, %k1 +; SKX-NEXT: vmovdqu64 (%rdi), %zmm0 {%k1} {z} +; SKX-NEXT: kshiftrw $8, %k1, %k1 +; SKX-NEXT: vmovdqu64 64(%rdi), %zmm1 {%k1} {z} +; SKX-NEXT: retq + %res = call <16 x %mystruct*> @llvm.masked.load.v16p0mystruct(<16 x %mystruct*>* %addr, i32 4, <16 x i1>%mask, <16 x %mystruct*>zeroinitializer) + ret <16 x %mystruct*> %res +} + +define void @test_store_16i64(<16 x i64>* %ptrs, <16 x i1> %mask, <16 x i64> %src0) { +; AVX512-LABEL: test_store_16i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpmovsxbd %xmm0, %zmm0 +; AVX512-NEXT: vpslld $31, %zmm0, %zmm0 +; AVX512-NEXT: vptestmd %zmm0, %zmm0, %k1 +; AVX512-NEXT: vmovdqu64 %zmm1, (%rdi) {%k1} +; AVX512-NEXT: kshiftrw $8, %k1, %k1 +; AVX512-NEXT: vmovdqu64 %zmm2, 64(%rdi) {%k1} +; AVX512-NEXT: retq +; +; AVX2-LABEL: test_store_16i64: +; AVX2: ## BB#0: +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm5 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm5, %xmm5 +; AVX2-NEXT: vpsrad $31, %xmm5, %xmm5 +; AVX2-NEXT: vpmovsxdq %xmm5, %ymm5 +; AVX2-NEXT: vpmaskmovq %ymm1, %ymm5, (%rdi) +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[3,1,2,3] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm1, %xmm1 +; AVX2-NEXT: vpsrad $31, %xmm1, %xmm1 +; AVX2-NEXT: vpmovsxdq %xmm1, %ymm1 +; AVX2-NEXT: vpmaskmovq %ymm4, %ymm1, 96(%rdi) +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm1, %xmm1 +; AVX2-NEXT: vpsrad $31, %xmm1, %xmm1 +; AVX2-NEXT: vpmovsxdq %xmm1, %ymm1 +; AVX2-NEXT: vpmaskmovq %ymm3, %ymm1, 64(%rdi) +; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm0, %xmm0 +; AVX2-NEXT: vpsrad $31, %xmm0, %xmm0 +; AVX2-NEXT: vpmovsxdq %xmm0, %ymm0 +; AVX2-NEXT: vpmaskmovq %ymm2, %ymm0, 32(%rdi) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; SKX-LABEL: test_store_16i64: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm0, %xmm0 +; SKX-NEXT: vpmovb2m %xmm0, %k1 +; SKX-NEXT: vmovdqu64 %zmm1, (%rdi) {%k1} +; SKX-NEXT: kshiftrw $8, %k1, %k1 +; SKX-NEXT: vmovdqu64 %zmm2, 64(%rdi) {%k1} +; SKX-NEXT: retq + call void @llvm.masked.store.v16i64(<16 x i64> %src0, <16 x i64>* %ptrs, i32 4, <16 x i1> %mask) + ret void +} +declare void @llvm.masked.store.v16i64(<16 x i64> %src0, <16 x i64>* %ptrs, i32, <16 x i1> %mask) +define void @test_store_16f64(<16 x double>* %ptrs, <16 x i1> %mask, <16 x double> %src0) { +; AVX512-LABEL: test_store_16f64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpmovsxbd %xmm0, %zmm0 +; AVX512-NEXT: vpslld $31, %zmm0, %zmm0 +; AVX512-NEXT: vptestmd %zmm0, %zmm0, %k1 +; AVX512-NEXT: vmovupd %zmm1, (%rdi) {%k1} +; AVX512-NEXT: kshiftrw $8, %k1, %k1 +; AVX512-NEXT: vmovupd %zmm2, 64(%rdi) {%k1} +; AVX512-NEXT: retq +; +; AVX2-LABEL: test_store_16f64: +; AVX2: ## BB#0: +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm5 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm5, %xmm5 +; AVX2-NEXT: vpsrad $31, %xmm5, %xmm5 +; AVX2-NEXT: vpmovsxdq %xmm5, %ymm5 +; AVX2-NEXT: vmaskmovpd %ymm1, %ymm5, (%rdi) +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[3,1,2,3] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm1, %xmm1 +; AVX2-NEXT: vpsrad $31, %xmm1, %xmm1 +; AVX2-NEXT: vpmovsxdq %xmm1, %ymm1 +; AVX2-NEXT: vmaskmovpd %ymm4, %ymm1, 96(%rdi) +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm1, %xmm1 +; AVX2-NEXT: vpsrad $31, %xmm1, %xmm1 +; AVX2-NEXT: vpmovsxdq %xmm1, %ymm1 +; AVX2-NEXT: vmaskmovpd %ymm3, %ymm1, 64(%rdi) +; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm0, %xmm0 +; AVX2-NEXT: vpsrad $31, %xmm0, %xmm0 +; AVX2-NEXT: vpmovsxdq %xmm0, %ymm0 +; AVX2-NEXT: vmaskmovpd %ymm2, %ymm0, 32(%rdi) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; SKX-LABEL: test_store_16f64: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm0, %xmm0 +; SKX-NEXT: vpmovb2m %xmm0, %k1 +; SKX-NEXT: vmovupd %zmm1, (%rdi) {%k1} +; SKX-NEXT: kshiftrw $8, %k1, %k1 +; SKX-NEXT: vmovupd %zmm2, 64(%rdi) {%k1} +; SKX-NEXT: retq + call void @llvm.masked.store.v16f64(<16 x double> %src0, <16 x double>* %ptrs, i32 4, <16 x i1> %mask) + ret void +} +declare void @llvm.masked.store.v16f64(<16 x double> %src0, <16 x double>* %ptrs, i32, <16 x i1> %mask) +define <16 x i64> @test_load_16i64(<16 x i64>* %ptrs, <16 x i1> %mask, <16 x i64> %src0) { +; AVX512-LABEL: test_load_16i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpmovsxbd %xmm0, %zmm0 +; AVX512-NEXT: vpslld $31, %zmm0, %zmm0 +; AVX512-NEXT: vptestmd %zmm0, %zmm0, %k1 +; AVX512-NEXT: vmovdqu64 (%rdi), %zmm1 {%k1} +; AVX512-NEXT: kshiftrw $8, %k1, %k1 +; AVX512-NEXT: vmovdqu64 64(%rdi), %zmm2 {%k1} +; AVX512-NEXT: vmovaps %zmm1, %zmm0 +; AVX512-NEXT: vmovaps %zmm2, %zmm1 +; AVX512-NEXT: retq +; +; AVX2-LABEL: test_load_16i64: +; AVX2: ## BB#0: +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm5 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm5, %xmm5 +; AVX2-NEXT: vpsrad $31, %xmm5, %xmm5 +; AVX2-NEXT: vpmovsxdq %xmm5, %ymm5 +; AVX2-NEXT: vpmaskmovq (%rdi), %ymm5, %ymm9 +; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[1,1,2,3] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm7, %xmm7 +; AVX2-NEXT: vpsrad $31, %xmm7, %xmm7 +; AVX2-NEXT: vpmovsxdq %xmm7, %ymm7 +; AVX2-NEXT: vpmaskmovq 32(%rdi), %ymm7, %ymm8 +; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[2,3,0,1] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm6, %xmm6 +; AVX2-NEXT: vpsrad $31, %xmm6, %xmm6 +; AVX2-NEXT: vpmovsxdq %xmm6, %ymm6 +; AVX2-NEXT: vpmaskmovq 64(%rdi), %ymm6, %ymm10 +; AVX2-NEXT: vblendvpd %ymm5, %ymm9, %ymm1, %ymm5 +; AVX2-NEXT: vblendvpd %ymm7, %ymm8, %ymm2, %ymm1 +; AVX2-NEXT: vblendvpd %ymm6, %ymm10, %ymm3, %ymm2 +; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm0, %xmm0 +; AVX2-NEXT: vpsrad $31, %xmm0, %xmm0 +; AVX2-NEXT: vpmovsxdq %xmm0, %ymm0 +; AVX2-NEXT: vpmaskmovq 96(%rdi), %ymm0, %ymm3 +; AVX2-NEXT: vblendvpd %ymm0, %ymm3, %ymm4, %ymm3 +; AVX2-NEXT: vmovapd %ymm5, %ymm0 +; AVX2-NEXT: retq +; +; SKX-LABEL: test_load_16i64: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm0, %xmm0 +; SKX-NEXT: vpmovb2m %xmm0, %k1 +; SKX-NEXT: vmovdqu64 (%rdi), %zmm1 {%k1} +; SKX-NEXT: kshiftrw $8, %k1, %k1 +; SKX-NEXT: vmovdqu64 64(%rdi), %zmm2 {%k1} +; SKX-NEXT: vmovaps %zmm1, %zmm0 +; SKX-NEXT: vmovaps %zmm2, %zmm1 +; SKX-NEXT: retq + %res = call <16 x i64> @llvm.masked.load.v16i64(<16 x i64>* %ptrs, i32 4, <16 x i1> %mask, <16 x i64> %src0) + ret <16 x i64> %res +} +declare <16 x i64> @llvm.masked.load.v16i64(<16 x i64>* %ptrs, i32, <16 x i1> %mask, <16 x i64> %src0) +define <16 x double> @test_load_16f64(<16 x double>* %ptrs, <16 x i1> %mask, <16 x double> %src0) { +; AVX512-LABEL: test_load_16f64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpmovsxbd %xmm0, %zmm0 +; AVX512-NEXT: vpslld $31, %zmm0, %zmm0 +; AVX512-NEXT: vptestmd %zmm0, %zmm0, %k1 +; AVX512-NEXT: vmovupd (%rdi), %zmm1 {%k1} +; AVX512-NEXT: kshiftrw $8, %k1, %k1 +; AVX512-NEXT: vmovupd 64(%rdi), %zmm2 {%k1} +; AVX512-NEXT: vmovaps %zmm1, %zmm0 +; AVX512-NEXT: vmovaps %zmm2, %zmm1 +; AVX512-NEXT: retq +; +; AVX2-LABEL: test_load_16f64: +; AVX2: ## BB#0: +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm5 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm5, %xmm5 +; AVX2-NEXT: vpsrad $31, %xmm5, %xmm5 +; AVX2-NEXT: vpmovsxdq %xmm5, %ymm5 +; AVX2-NEXT: vmaskmovpd (%rdi), %ymm5, %ymm9 +; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[1,1,2,3] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm7, %xmm7 +; AVX2-NEXT: vpsrad $31, %xmm7, %xmm7 +; AVX2-NEXT: vpmovsxdq %xmm7, %ymm7 +; AVX2-NEXT: vmaskmovpd 32(%rdi), %ymm7, %ymm8 +; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[2,3,0,1] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm6, %xmm6 +; AVX2-NEXT: vpsrad $31, %xmm6, %xmm6 +; AVX2-NEXT: vpmovsxdq %xmm6, %ymm6 +; AVX2-NEXT: vmaskmovpd 64(%rdi), %ymm6, %ymm10 +; AVX2-NEXT: vblendvpd %ymm5, %ymm9, %ymm1, %ymm5 +; AVX2-NEXT: vblendvpd %ymm7, %ymm8, %ymm2, %ymm1 +; AVX2-NEXT: vblendvpd %ymm6, %ymm10, %ymm3, %ymm2 +; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm0, %xmm0 +; AVX2-NEXT: vpsrad $31, %xmm0, %xmm0 +; AVX2-NEXT: vpmovsxdq %xmm0, %ymm0 +; AVX2-NEXT: vmaskmovpd 96(%rdi), %ymm0, %ymm3 +; AVX2-NEXT: vblendvpd %ymm0, %ymm3, %ymm4, %ymm3 +; AVX2-NEXT: vmovapd %ymm5, %ymm0 +; AVX2-NEXT: retq +; +; SKX-LABEL: test_load_16f64: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %xmm0, %xmm0 +; SKX-NEXT: vpmovb2m %xmm0, %k1 +; SKX-NEXT: vmovupd (%rdi), %zmm1 {%k1} +; SKX-NEXT: kshiftrw $8, %k1, %k1 +; SKX-NEXT: vmovupd 64(%rdi), %zmm2 {%k1} +; SKX-NEXT: vmovaps %zmm1, %zmm0 +; SKX-NEXT: vmovaps %zmm2, %zmm1 +; SKX-NEXT: retq + %res = call <16 x double> @llvm.masked.load.v16f64(<16 x double>* %ptrs, i32 4, <16 x i1> %mask, <16 x double> %src0) + ret <16 x double> %res +} +declare <16 x double> @llvm.masked.load.v16f64(<16 x double>* %ptrs, i32, <16 x i1> %mask, <16 x double> %src0) + +define <32 x double> @test_load_32f64(<32 x double>* %ptrs, <32 x i1> %mask, <32 x double> %src0) { +; AVX512-LABEL: test_load_32f64: +; AVX512: ## BB#0: +; AVX512-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX512-NEXT: vpmovsxbd %xmm5, %zmm5 +; AVX512-NEXT: vpslld $31, %zmm5, %zmm5 +; AVX512-NEXT: vptestmd %zmm5, %zmm5, %k1 +; AVX512-NEXT: vmovupd 128(%rdi), %zmm3 {%k1} +; AVX512-NEXT: vpmovsxbd %xmm0, %zmm0 +; AVX512-NEXT: vpslld $31, %zmm0, %zmm0 +; AVX512-NEXT: vptestmd %zmm0, %zmm0, %k2 +; AVX512-NEXT: vmovupd (%rdi), %zmm1 {%k2} +; AVX512-NEXT: kshiftrw $8, %k1, %k1 +; AVX512-NEXT: vmovupd 192(%rdi), %zmm4 {%k1} +; AVX512-NEXT: kshiftrw $8, %k2, %k1 +; AVX512-NEXT: vmovupd 64(%rdi), %zmm2 {%k1} +; AVX512-NEXT: vmovaps %zmm1, %zmm0 +; AVX512-NEXT: vmovaps %zmm2, %zmm1 +; AVX512-NEXT: vmovaps %zmm3, %zmm2 +; AVX512-NEXT: vmovaps %zmm4, %zmm3 +; AVX512-NEXT: retq +; +; AVX2-LABEL: test_load_32f64: +; AVX2: ## BB#0: +; AVX2-NEXT: pushq %rbp +; AVX2-NEXT: Ltmp0: +; AVX2-NEXT: .cfi_def_cfa_offset 16 +; AVX2-NEXT: Ltmp1: +; AVX2-NEXT: .cfi_offset %rbp, -16 +; AVX2-NEXT: movq %rsp, %rbp +; AVX2-NEXT: Ltmp2: +; AVX2-NEXT: .cfi_def_cfa_register %rbp +; AVX2-NEXT: andq $-32, %rsp +; AVX2-NEXT: subq $32, %rsp +; AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[1,1,2,3] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm8, %xmm8 +; AVX2-NEXT: vpsrad $31, %xmm8, %xmm8 +; AVX2-NEXT: vpmovsxdq %xmm8, %ymm8 +; AVX2-NEXT: vmaskmovpd 32(%rsi), %ymm8, %ymm9 +; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm0[2,3,0,1] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero,xmm10[2],zero,zero,zero,xmm10[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm10, %xmm10 +; AVX2-NEXT: vpsrad $31, %xmm10, %xmm10 +; AVX2-NEXT: vpmovsxdq %xmm10, %ymm10 +; AVX2-NEXT: vmaskmovpd 64(%rsi), %ymm10, %ymm11 +; AVX2-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[3,1,2,3] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm12 = xmm12[0],zero,zero,zero,xmm12[1],zero,zero,zero,xmm12[2],zero,zero,zero,xmm12[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm12, %xmm12 +; AVX2-NEXT: vpsrad $31, %xmm12, %xmm12 +; AVX2-NEXT: vpmovsxdq %xmm12, %ymm12 +; AVX2-NEXT: vmaskmovpd 96(%rsi), %ymm12, %ymm13 +; AVX2-NEXT: vblendvpd %ymm8, %ymm9, %ymm2, %ymm8 +; AVX2-NEXT: vblendvpd %ymm10, %ymm11, %ymm3, %ymm9 +; AVX2-NEXT: vblendvpd %ymm12, %ymm13, %ymm4, %ymm11 +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2 +; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[1,1,2,3] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm3, %xmm3 +; AVX2-NEXT: vpsrad $31, %xmm3, %xmm3 +; AVX2-NEXT: vpmovsxdq %xmm3, %ymm3 +; AVX2-NEXT: vmaskmovpd 160(%rsi), %ymm3, %ymm10 +; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,0,1] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm4, %xmm4 +; AVX2-NEXT: vpsrad $31, %xmm4, %xmm4 +; AVX2-NEXT: vpmovsxdq %xmm4, %ymm4 +; AVX2-NEXT: vmaskmovpd 192(%rsi), %ymm4, %ymm12 +; AVX2-NEXT: vblendvpd %ymm3, %ymm10, %ymm6, %ymm3 +; AVX2-NEXT: vmovapd 16(%rbp), %ymm6 +; AVX2-NEXT: vblendvpd %ymm4, %ymm12, %ymm7, %ymm4 +; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm2[3,1,2,3] +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm7, %xmm7 +; AVX2-NEXT: vpsrad $31, %xmm7, %xmm7 +; AVX2-NEXT: vpmovsxdq %xmm7, %ymm7 +; AVX2-NEXT: vmaskmovpd 224(%rsi), %ymm7, %ymm10 +; AVX2-NEXT: vblendvpd %ymm7, %ymm10, %ymm6, %ymm6 +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm0, %xmm0 +; AVX2-NEXT: vpsrad $31, %xmm0, %xmm0 +; AVX2-NEXT: vpmovsxdq %xmm0, %ymm0 +; AVX2-NEXT: vmaskmovpd (%rsi), %ymm0, %ymm7 +; AVX2-NEXT: vblendvpd %ymm0, %ymm7, %ymm1, %ymm0 +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero +; AVX2-NEXT: vpslld $31, %xmm1, %xmm1 +; AVX2-NEXT: vpsrad $31, %xmm1, %xmm1 +; AVX2-NEXT: vpmovsxdq %xmm1, %ymm1 +; AVX2-NEXT: vmaskmovpd 128(%rsi), %ymm1, %ymm2 +; AVX2-NEXT: vblendvpd %ymm1, %ymm2, %ymm5, %ymm1 +; AVX2-NEXT: vmovapd %ymm1, 128(%rdi) +; AVX2-NEXT: vmovapd %ymm0, (%rdi) +; AVX2-NEXT: vmovapd %ymm6, 224(%rdi) +; AVX2-NEXT: vmovapd %ymm4, 192(%rdi) +; AVX2-NEXT: vmovapd %ymm3, 160(%rdi) +; AVX2-NEXT: vmovapd %ymm11, 96(%rdi) +; AVX2-NEXT: vmovapd %ymm9, 64(%rdi) +; AVX2-NEXT: vmovapd %ymm8, 32(%rdi) +; AVX2-NEXT: movq %rdi, %rax +; AVX2-NEXT: movq %rbp, %rsp +; AVX2-NEXT: popq %rbp +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; SKX-LABEL: test_load_32f64: +; SKX: ## BB#0: +; SKX-NEXT: vpsllw $7, %ymm0, %ymm0 +; SKX-NEXT: vpmovb2m %ymm0, %k1 +; SKX-NEXT: vmovupd (%rdi), %zmm1 {%k1} +; SKX-NEXT: kshiftrd $16, %k1, %k2 +; SKX-NEXT: vmovupd 128(%rdi), %zmm3 {%k2} +; SKX-NEXT: kshiftrw $8, %k1, %k1 +; SKX-NEXT: vmovupd 64(%rdi), %zmm2 {%k1} +; SKX-NEXT: kshiftrw $8, %k2, %k1 +; SKX-NEXT: vmovupd 192(%rdi), %zmm4 {%k1} +; SKX-NEXT: vmovaps %zmm1, %zmm0 +; SKX-NEXT: vmovaps %zmm2, %zmm1 +; SKX-NEXT: vmovaps %zmm3, %zmm2 +; SKX-NEXT: vmovaps %zmm4, %zmm3 +; SKX-NEXT: retq + %res = call <32 x double> @llvm.masked.load.v32f64(<32 x double>* %ptrs, i32 4, <32 x i1> %mask, <32 x double> %src0) + ret <32 x double> %res +} +declare <32 x double> @llvm.masked.load.v32f64(<32 x double>* %ptrs, i32, <32 x i1> %mask, <32 x double> %src0) diff --git a/test/CodeGen/X86/materialize.ll b/test/CodeGen/X86/materialize.ll new file mode 100644 index 000000000000..695bf0fa5b98 --- /dev/null +++ b/test/CodeGen/X86/materialize.ll @@ -0,0 +1,184 @@ +; RUN: llc -mtriple=i686-unknown-linux-gnu -mattr=+cmov %s -o - | FileCheck %s --check-prefix=CHECK32 +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+cmov %s -o - | FileCheck %s --check-prefix=CHECK64 +; RUN: llc -mtriple=x86_64-pc-win32 -mattr=+cmov %s -o - | FileCheck %s --check-prefix=CHECKWIN64 + +define i32 @one32_nooptsize() { +entry: + ret i32 1 + +; When not optimizing for size, use mov. +; CHECK32-LABEL: one32_nooptsize: +; CHECK32: movl $1, %eax +; CHECK32-NEXT: retl +; CHECK64-LABEL: one32_nooptsize: +; CHECK64: movl $1, %eax +; CHECK64-NEXT: retq +} + +define i32 @one32() optsize { +entry: + ret i32 1 + +; CHECK32-LABEL: one32: +; CHECK32: xorl %eax, %eax +; CHECK32-NEXT: incl %eax +; CHECK32-NEXT: retl + +; FIXME: Figure out the best approach in 64-bit mode. +; CHECK64-LABEL: one32: +; CHECK64: movl $1, %eax +; CHECK64-NEXT: retq +} + +define i32 @one32_minsize() minsize { +entry: + ret i32 1 + +; On 32-bit, xor-inc is preferred over push-pop. +; CHECK32-LABEL: one32_minsize: +; CHECK32: xorl %eax, %eax +; CHECK32-NEXT: incl %eax +; CHECK32-NEXT: retl + +; On 64-bit we don't do xor-inc yet, so push-pop it is. Note that we have to +; pop into a 64-bit register even when we just need 32 bits. +; CHECK64-LABEL: one32_minsize: +; CHECK64: pushq $1 +; CHECK64: .cfi_adjust_cfa_offset 8 +; CHECK64: popq %rax +; CHECK64: .cfi_adjust_cfa_offset -8 +; CHECK64-NEXT: retq +} + +define i64 @one64_minsize() minsize { +entry: + ret i64 1 +; On 64-bit we don't do xor-inc yet, so push-pop it is. +; CHECK64-LABEL: one64_minsize: +; CHECK64: pushq $1 +; CHECK64: .cfi_adjust_cfa_offset 8 +; CHECK64: popq %rax +; CHECK64: .cfi_adjust_cfa_offset -8 +; CHECK64-NEXT: retq + +; On Win64 we can't adjust the stack unless there's a frame pointer. +; CHECKWIN64-LABEL: one64_minsize: +; CHECKWIN64: movl $1, %eax +; CHECKWIN64-NEXT: retq +} + +define i32 @minus_one32() optsize { +entry: + ret i32 -1 + +; CHECK32-LABEL: minus_one32: +; CHECK32: xorl %eax, %eax +; CHECK32-NEXT: decl %eax +; CHECK32-NEXT: retl +} + +define i32 @minus_one32_minsize() minsize { +entry: + ret i32 -1 + +; xor-dec is preferred over push-pop. +; CHECK32-LABEL: minus_one32_minsize: +; CHECK32: xorl %eax, %eax +; CHECK32-NEXT: decl %eax +; CHECK32-NEXT: retl +} + +define i16 @one16() optsize { +entry: + ret i16 1 + +; CHECK32-LABEL: one16: +; CHECK32: xorl %eax, %eax +; CHECK32-NEXT: incl %eax +; CHECK32-NEXT: retl +} + +define i16 @minus_one16() optsize { +entry: + ret i16 -1 + +; CHECK32-LABEL: minus_one16: +; CHECK32: xorl %eax, %eax +; CHECK32-NEXT: decl %eax +; CHECK32-NEXT: retl +} + +define i32 @minus_five32() minsize { +entry: + ret i32 -5 + +; CHECK32-LABEL: minus_five32: +; CHECK32: pushl $-5 +; CHECK32: popl %eax +; CHECK32: retl +} + +define i64 @minus_five64() minsize { +entry: + ret i64 -5 + +; CHECK64-LABEL: minus_five64: +; CHECK64: pushq $-5 +; CHECK64: .cfi_adjust_cfa_offset 8 +; CHECK64: popq %rax +; CHECK64: .cfi_adjust_cfa_offset -8 +; CHECK64: retq +} + +define i32 @rematerialize_minus_one() optsize { +entry: + ; Materialize -1 (thiscall forces it into %ecx). + tail call x86_thiscallcc void @f(i32 -1) + + ; Clobber all registers except %esp, leaving nowhere to store the -1 besides + ; spilling it to the stack. + tail call void asm sideeffect "", "~{eax},~{ebx},~{ecx},~{edx},~{edi},~{esi},~{ebp},~{dirflag},~{fpsr},~{flags}"() + + ; -1 should be re-materialized here instead of getting spilled above. + ret i32 -1 + +; CHECK32-LABEL: rematerialize_minus_one +; CHECK32: xorl %ecx, %ecx +; CHECK32-NEXT: decl %ecx +; CHECK32: calll +; CHECK32: xorl %eax, %eax +; CHECK32-NEXT: decl %eax +; CHECK32-NOT: %eax +; CHECK32: retl +} + +define i32 @rematerialize_minus_one_eflags(i32 %x) optsize { +entry: + ; Materialize -1 (thiscall forces it into %ecx). + tail call x86_thiscallcc void @f(i32 -1) + + ; Clobber all registers except %esp, leaving nowhere to store the -1 besides + ; spilling it to the stack. + tail call void asm sideeffect "", "~{eax},~{ebx},~{ecx},~{edx},~{edi},~{esi},~{ebp},~{dirflag},~{fpsr},~{flags}"() + + ; Define eflags. + %a = icmp ne i32 %x, 123 + %b = zext i1 %a to i32 + ; Cause -1 to be rematerialized right in front of the cmov, which needs eflags. + ; It must therefore not use the xor-dec lowering. + %c = select i1 %a, i32 %b, i32 -1 + ret i32 %c + +; CHECK32-LABEL: rematerialize_minus_one_eflags +; CHECK32: xorl %ecx, %ecx +; CHECK32-NEXT: decl %ecx +; CHECK32: calll +; CHECK32: cmpl +; CHECK32: setne +; CHECK32-NOT: xorl +; CHECK32: movl $-1 +; CHECK32: cmov +; CHECK32: retl +} + +declare x86_thiscallcc void @f(i32) diff --git a/test/CodeGen/X86/mcu-abi.ll b/test/CodeGen/X86/mcu-abi.ll new file mode 100644 index 000000000000..966fd4521f2d --- /dev/null +++ b/test/CodeGen/X86/mcu-abi.ll @@ -0,0 +1,112 @@ +; RUN: llc < %s -mtriple=i686-pc-elfiamcu | FileCheck %s + +%struct.st12_t = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } + +; CHECK-LABEL: test_ints: +; CHECK: addl %edx, %eax +; CHECK-NEXT: imull %ecx, %eax +; CHECK-NEXT: addl 4(%esp), %eax +; CHECK-NEXT: retl +define i32 @test_ints(i32 %a, i32 %b, i32 %c, i32 %d) #0 { +entry: + %r1 = add i32 %b, %a + %r2 = mul i32 %c, %r1 + %r3 = add i32 %d, %r2 + ret i32 %r3 +} + +; CHECK-LABEL: test_floats: +; CHECK: addl %edx, %eax +; CHECK-NEXT: imull %ecx, %eax +; CHECK-NEXT: addl 4(%esp), %eax +; CHECK-NEXT: retl +define i32 @test_floats(i32 %a, i32 %b, float %c, float %d) #0 { +entry: + %ci = bitcast float %c to i32 + %di = bitcast float %d to i32 + %r1 = add i32 %b, %a + %r2 = mul i32 %ci, %r1 + %r3 = add i32 %di, %r2 + ret i32 %r3 +} + +; CHECK-LABEL: test_doubles: +; CHECK: addl 4(%esp), %eax +; CHECK-NEXT: adcl 8(%esp), %edx +; CHECK-NEXT: retl +define double @test_doubles(double %d1, double %d2) #0 { +entry: + %d1i = bitcast double %d1 to i64 + %d2i = bitcast double %d2 to i64 + %r = add i64 %d1i, %d2i + %rd = bitcast i64 %r to double + ret double %rd +} + +; CHECK-LABEL: test_mixed_doubles: +; CHECK: addl %ecx, %eax +; CHECK-NEXT: adcl $0, %edx +; CHECK-NEXT: retl +define double @test_mixed_doubles(double %d2, i32 %i) #0 { +entry: + %iext = zext i32 %i to i64 + %d2i = bitcast double %d2 to i64 + %r = add i64 %iext, %d2i + %rd = bitcast i64 %r to double + ret double %rd +} + +; CHECK-LABEL: ret_large_struct: +; CHECK: pushl %esi +; CHECK-NEXT: movl %eax, %esi +; CHECK-NEXT: leal 8(%esp), %edx +; CHECK-NEXT: movl $48, %ecx +; CHECK-NEXT: calll memcpy +; CHECK-NEXT: movl %esi, %eax +; CHECK-NEXT: popl %esi +; CHECK-NOT: retl $4 +; CHECK-NEXT: retl +define void @ret_large_struct(%struct.st12_t* noalias nocapture sret %agg.result, %struct.st12_t* byval nocapture readonly align 4 %r) #0 { +entry: + %0 = bitcast %struct.st12_t* %agg.result to i8* + %1 = bitcast %struct.st12_t* %r to i8* + call void @llvm.memcpy.p0i8.p0i8.i32(i8* %0, i8* %1, i32 48, i32 1, i1 false) + ret void +} + +; CHECK-LABEL: var_args: +; CHECK: movl 4(%esp), %eax +; CHECK-NEXT: retl +define i32 @var_args(i32 %i1, ...) #0 { +entry: + ret i32 %i1 +} + +; CHECK-LABEL: test_lib_args: +; CHECK: movl %edx, %eax +; CHECK: calll __fixsfsi +define i32 @test_lib_args(float %a, float %b) #0 { + %ret = fptosi float %b to i32 + ret i32 %ret +} + +; CHECK-LABEL: test_fp128: +; CHECK: movl (%eax), %e[[CX:..]] +; CHECK-NEXT: movl 4(%eax), %e[[DX:..]] +; CHECK-NEXT: movl 8(%eax), %e[[SI:..]] +; CHECK-NEXT: movl 12(%eax), %e[[AX:..]] +; CHECK-NEXT: movl %e[[AX]], 12(%esp) +; CHECK-NEXT: movl %e[[SI]], 8(%esp) +; CHECK-NEXT: movl %e[[DX]], 4(%esp) +; CHECK-NEXT: movl %e[[CX]], (%esp) +; CHECK-NEXT: calll __fixtfsi +define i32 @test_fp128(fp128* %ptr) #0 { + %v = load fp128, fp128* %ptr + %ret = fptosi fp128 %v to i32 + ret i32 %ret +} + +declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i32, i1) #1 + +attributes #0 = { nounwind "use-soft-float"="true"} +attributes #1 = { nounwind argmemonly } diff --git a/test/CodeGen/X86/memcpy-2.ll b/test/CodeGen/X86/memcpy-2.ll index 0111c0d433f1..7ef61c9a677b 100644 --- a/test/CodeGen/X86/memcpy-2.ll +++ b/test/CodeGen/X86/memcpy-2.ll @@ -3,6 +3,8 @@ ; RUN: llc < %s -mattr=+sse,-sse2 -mtriple=i686-apple-darwin -mcpu=core2 | FileCheck %s -check-prefix=SSE1 ; RUN: llc < %s -mattr=-sse -mtriple=i686-apple-darwin -mcpu=core2 | FileCheck %s -check-prefix=NOSSE ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=core2 | FileCheck %s -check-prefix=X86-64 +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=nehalem | FileCheck %s -check-prefix=NHM_64 + @.str = internal constant [25 x i8] c"image\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00" @.str2 = internal constant [30 x i8] c"xxxxxxxxxxxxxxxxxxxxxxxxxxxxx\00", align 4 @@ -179,13 +181,25 @@ entry: ; NOSSE: movl $2021161080 ; NOSSE: movl $2021161080 +;;; TODO: (1) Some of the loads and stores are certainly unaligned and (2) the first load and first +;;; store overlap with the second load and second store respectively. +;;; +;;; Is either of the sequences ideal? + ; X86-64-LABEL: t4: -; X86-64: movabsq $8680820740569200760, %rax -; X86-64: movq %rax -; X86-64: movq %rax -; X86-64: movq %rax -; X86-64: movw $120 -; X86-64: movl $2021161080 +; X86-64: movabsq $33909456017848440, %rax ## imm = 0x78787878787878 +; X86-64: movq %rax, -10(%rsp) +; X86-64: movabsq $8680820740569200760, %rax ## imm = 0x7878787878787878 +; X86-64: movq %rax, -16(%rsp) +; X86-64: movq %rax, -24(%rsp) +; X86-64: movq %rax, -32(%rsp) + +; NHM_64-LABEL: t4: +; NHM_64: movups _.str2+14(%rip), %xmm0 +; NHM_64: movups %xmm0, -26(%rsp) +; NHM_64: movups _.str2(%rip), %xmm0 +; NHM_64: movaps %xmm0, -40(%rsp) + %tmp1 = alloca [30 x i8] %tmp2 = bitcast [30 x i8]* %tmp1 to i8* call void @llvm.memcpy.p0i8.p0i8.i32(i8* %tmp2, i8* getelementptr inbounds ([30 x i8], [30 x i8]* @.str2, i32 0, i32 0), i32 30, i32 1, i1 false) diff --git a/test/CodeGen/X86/memcpy.ll b/test/CodeGen/X86/memcpy.ll index f582571252b5..4351014192bb 100644 --- a/test/CodeGen/X86/memcpy.ll +++ b/test/CodeGen/X86/memcpy.ll @@ -2,6 +2,7 @@ ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=core2 | FileCheck %s -check-prefix=DARWIN declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1) nounwind +declare void @llvm.memcpy.p256i8.p256i8.i64(i8 addrspace(256)* nocapture, i8 addrspace(256)* nocapture, i64, i32, i1) nounwind ; Variable memcpy's should lower to calls. @@ -59,6 +60,26 @@ entry: ; DARWIN: movq } +define void @test3_minsize(i8* nocapture %A, i8* nocapture %B) nounwind minsize noredzone { + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %A, i8* %B, i64 64, i32 1, i1 false) + ret void +; LINUX-LABEL: test3_minsize: +; LINUX: memcpy + +; DARWIN-LABEL: test3_minsize: +; DARWIN: memcpy +} + +define void @test3_minsize_optsize(i8* nocapture %A, i8* nocapture %B) nounwind optsize minsize noredzone { + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %A, i8* %B, i64 64, i32 1, i1 false) + ret void +; LINUX-LABEL: test3_minsize_optsize: +; LINUX: memcpy + +; DARWIN-LABEL: test3_minsize_optsize: +; DARWIN: memcpy +} + ; Large constant memcpy's should be inlined when not optimizing for size. define void @test4(i8* nocapture %A, i8* nocapture %B) nounwind noredzone { entry: @@ -118,3 +139,15 @@ define void @PR15348(i8* %a, i8* %b) { call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a, i8* %b, i64 17, i32 0, i1 false) ret void } + +; Memcpys from / to address space 256 should be lowered to appropriate loads / +; stores if small enough. +define void @addrspace256(i8 addrspace(256)* %a, i8 addrspace(256)* %b) nounwind { + tail call void @llvm.memcpy.p256i8.p256i8.i64(i8 addrspace(256)* %a, i8 addrspace(256)* %b, i64 16, i32 8, i1 false) + ret void +; LINUX-LABEL: addrspace256: +; LINUX: movq %gs: +; LINUX: movq %gs: +; LINUX: movq {{.*}}, %gs: +; LINUX: movq {{.*}}, %gs: +} diff --git a/test/CodeGen/X86/merge-store-partially-alias-loads.ll b/test/CodeGen/X86/merge-store-partially-alias-loads.ll new file mode 100644 index 000000000000..8e148aa76d38 --- /dev/null +++ b/test/CodeGen/X86/merge-store-partially-alias-loads.ll @@ -0,0 +1,52 @@ +; REQUIRES: asserts +; RUN: llc -march=x86-64 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck -check-prefix=X86 %s +; RUN: llc -march=x86-64 -mtriple=x86_64-unknown-linux-gnu -debug-only=isel < %s 2>&1 | FileCheck -check-prefix=DBGDAG %s + +; It's OK to merge the load / store of the first 2 components, but +; they must not be placed on the same chain after merging. + +; X86-LABEL: {{^}}merge_store_partial_overlap_load: +; X86-DAG: movw ([[BASEREG:%[a-z]+]]), [[LO2:%[a-z]+]] +; X86-DAG: movb 2([[BASEREG]]), [[HI1:%[a-z]+]] + +; X86-NEXT: movw [[LO2]], 1([[BASEREG]]) +; X86-NEXT: movb [[HI1]], 3([[BASEREG]]) +; X86-NEXT: retq + +; DBGDAG-LABEL: Optimized lowered selection DAG: BB#0 'merge_store_partial_overlap_load:' +; DBGDAG: [[ENTRYTOKEN:t[0-9]+]]: ch = EntryToken +; DBGDAG-DAG: [[BASEPTR:t[0-9]+]]: i64,ch = CopyFromReg [[ENTRYTOKEN]], +; DBGDAG-DAG: [[ADDPTR:t[0-9]+]]: i64 = add [[BASEPTR]], Constant:i64<2> + +; DBGDAG-DAG: [[LD2:t[0-9]+]]: i16,ch = load [[ENTRYTOKEN]], [[BASEPTR]], undef:i64 +; DBGDAG-DAG: [[LD1:t[0-9]+]]: i8,ch = load [[ENTRYTOKEN]], [[ADDPTR]], undef:i64 + +; DBGDAG: [[LOADTOKEN:t[0-9]+]]: ch = TokenFactor [[LD2]]:1, [[LD1]]:1 + +; DBGDAG-DAG: [[ST2:t[0-9]+]]: ch = store [[LOADTOKEN]], [[LD2]], t{{[0-9]+}}, undef:i64 +; DBGDAG-DAG: [[ST1:t[0-9]+]]: ch = store [[ST2]], [[LD1]], t{{[0-9]+}}, undef:i64 +; DBGDAG: X86ISD::RET_FLAG [[ST1]], + +; DBGDAG: Type-legalized selection DAG: BB#0 'merge_store_partial_overlap_load:' +define void @merge_store_partial_overlap_load([4 x i8]* %tmp) { + %tmp8 = getelementptr inbounds [4 x i8], [4 x i8]* %tmp, i32 0, i8 0 + %tmp10 = getelementptr inbounds [4 x i8], [4 x i8]* %tmp, i32 0, i8 1 + %tmp12 = getelementptr inbounds [4 x i8], [4 x i8]* %tmp, i32 0, i8 2 + %tmp14 = getelementptr [4 x i8], [4 x i8]* %tmp, i32 0, i8 3 + + %tmp9 = load i8, i8* %tmp8, align 1 ; base + 0 + %tmp11 = load i8, i8* %tmp10, align 1 ; base + 1 + %tmp13 = load i8, i8* %tmp12, align 1 ; base + 2 + + store i8 %tmp9, i8* %tmp10, align 1 ; base + 1 + store i8 %tmp11, i8* %tmp12, align 1 ; base + 2 + store i8 %tmp13, i8* %tmp14, align 1 ; base + 3 + +; Should emit +; load base + 0, base + 1 +; store base + 1, base + 2 +; load base + 2 +; store base + 3 + + ret void +} diff --git a/test/CodeGen/X86/misched-code-difference-with-debug.ll b/test/CodeGen/X86/misched-code-difference-with-debug.ll index 0f1f382c49a8..0a1ea830a41d 100644 --- a/test/CodeGen/X86/misched-code-difference-with-debug.ll +++ b/test/CodeGen/X86/misched-code-difference-with-debug.ll @@ -43,7 +43,7 @@ entry: ; CHECK-LABEL: test_with_debug ; CHECK: movl [[A]], [[B]] ; CHECK-NEXT: movl [[A]], [[C]] -define void @test_with_debug() { +define void @test_with_debug() !dbg !13 { entry: %c = alloca %class.C, align 1 %0 = load i8, i8* @argc, align 1 @@ -62,26 +62,26 @@ declare void @llvm.dbg.value(metadata, i64, metadata, metadata) !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!22, !23} -!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, enums: !2, retainedTypes: !3, subprograms: !12, globals: !20, imports: !2) +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, enums: !2, retainedTypes: !3, subprograms: !12, globals: !20, imports: !2) !1 = !DIFile(filename: "test.cpp", directory: "") !2 = !{} !3 = !{!4} !4 = !DICompositeType(tag: DW_TAG_class_type, name: "C", line: 2, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS1C") !5 = !{!6} -!6 = !DISubprogram(name: "test", file: !1, scope: !"_ZTS1C", type: !7) +!6 = !DISubprogram(name: "test", file: !1, scope: !"_ZTS1C", type: !7, isDefinition: false) !7 = !DISubroutineType(types: !8) !8 = !{!9, !10, !11, !11, !11, null} !9 = !DIBasicType(encoding: DW_ATE_signed, size: 32, align: 32, name: "int") !10 = !DIDerivedType(baseType: !"_ZTS1C", tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial) !11 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char) !12 = !{!13} -!13 = !DISubprogram(name: "test_with_debug", linkageName: "test_with_debug", line: 6, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !1, scope: !14, type: !15, function: void ()* @test_with_debug, variables: !17) +!13 = distinct !DISubprogram(name: "test_with_debug", linkageName: "test_with_debug", line: 6, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 6, file: !1, scope: !14, type: !15, variables: !17) !14 = !DIFile(filename: "test.cpp", directory: "") !15 = !DISubroutineType(types: !16) !16 = !{null} !17 = !{!18, !19} -!18 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 7, scope: !13, file: !14, type: !"_ZTS1C") -!19 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "lc", line: 8, scope: !13, file: !14, type: !11) +!18 = !DILocalVariable(name: "c", line: 7, scope: !13, file: !14, type: !"_ZTS1C") +!19 = !DILocalVariable(name: "lc", line: 8, scope: !13, file: !14, type: !11) !20 = !{!21} !21 = !DIGlobalVariable(name: "argc", line: 1, isLocal: false, isDefinition: true, scope: null, file: !14, type: !11, variable: i8* @argc) !22 = !{i32 2, !"Dwarf Version", i32 4} diff --git a/test/CodeGen/X86/mmx-arg-passing-x86-64.ll b/test/CodeGen/X86/mmx-arg-passing-x86-64.ll index 2727e3eb0280..9841381b560d 100644 --- a/test/CodeGen/X86/mmx-arg-passing-x86-64.ll +++ b/test/CodeGen/X86/mmx-arg-passing-x86-64.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+mmx,+sse2 | FileCheck %s --check-prefix=X86-64 ; ; On Darwin x86-64, v8i8, v4i16, v2i32 values are passed in XMM[0-7]. diff --git a/test/CodeGen/X86/mmx-arg-passing.ll b/test/CodeGen/X86/mmx-arg-passing.ll index 4e0031076200..67ccb9e32dde 100644 --- a/test/CodeGen/X86/mmx-arg-passing.ll +++ b/test/CodeGen/X86/mmx-arg-passing.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=i386-apple-darwin -mattr=+mmx | FileCheck %s --check-prefix=X86-32 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+mmx,+sse2 | FileCheck %s --check-prefix=X86-64 ; diff --git a/test/CodeGen/X86/mmx-coalescing.ll b/test/CodeGen/X86/mmx-coalescing.ll new file mode 100644 index 000000000000..a515e5ee3754 --- /dev/null +++ b/test/CodeGen/X86/mmx-coalescing.ll @@ -0,0 +1,84 @@ +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+mmx,+sse2 | FileCheck %s + +%SA = type <{ %union.anon, i32, [4 x i8], i8*, i8*, i8*, i32, [4 x i8] }> +%union.anon = type { <1 x i64> } + +; Check that extra movd (copy) instructions aren't generated. + +define i32 @test(%SA* %pSA, i16* %A, i32 %B, i32 %C, i32 %D, i8* %E) { +entry: +; CHECK-LABEL: test +; CHECK: # BB#0: +; CHECK-NEXT: pshufw +; CHECK-NEXT: movd +; CHECK-NOT: movd +; CHECK-NEXT: testl + %shl = shl i32 1, %B + %shl1 = shl i32 %C, %B + %shl2 = shl i32 1, %D + %v = getelementptr inbounds %SA, %SA* %pSA, i64 0, i32 0, i32 0 + %v0 = load <1 x i64>, <1 x i64>* %v, align 8 + %SA0 = getelementptr inbounds %SA, %SA* %pSA, i64 0, i32 1 + %v1 = load i32, i32* %SA0, align 4 + %SA1 = getelementptr inbounds %SA, %SA* %pSA, i64 0, i32 3 + %v2 = load i8*, i8** %SA1, align 8 + %SA2 = getelementptr inbounds %SA, %SA* %pSA, i64 0, i32 4 + %v3 = load i8*, i8** %SA2, align 8 + %v4 = bitcast <1 x i64> %v0 to <4 x i16> + %v5 = bitcast <4 x i16> %v4 to x86_mmx + %v6 = tail call x86_mmx @llvm.x86.sse.pshuf.w(x86_mmx %v5, i8 -18) + %v7 = bitcast x86_mmx %v6 to <4 x i16> + %v8 = bitcast <4 x i16> %v7 to <1 x i64> + %v9 = extractelement <1 x i64> %v8, i32 0 + %v10 = bitcast i64 %v9 to <2 x i32> + %v11 = extractelement <2 x i32> %v10, i32 0 + %cmp = icmp eq i32 %v11, 0 + br i1 %cmp, label %if.A, label %if.B + +if.A: +; CHECK: %if.A +; CHECK-NEXT: movd +; CHECK-NEXT: psllq + %pa = phi <1 x i64> [ %v8, %entry ], [ %vx, %if.C ] + %v17 = extractelement <1 x i64> %pa, i32 0 + %v18 = bitcast i64 %v17 to x86_mmx + %v19 = tail call x86_mmx @llvm.x86.mmx.pslli.q(x86_mmx %v18, i32 %B) #2 + %v20 = bitcast x86_mmx %v19 to i64 + %v21 = insertelement <1 x i64> undef, i64 %v20, i32 0 + %cmp3 = icmp eq i64 %v20, 0 + br i1 %cmp3, label %if.C, label %merge + +if.B: + %v34 = bitcast <1 x i64> %v8 to <4 x i16> + %v35 = bitcast <4 x i16> %v34 to x86_mmx + %v36 = tail call x86_mmx @llvm.x86.sse.pshuf.w(x86_mmx %v35, i8 -18) + %v37 = bitcast x86_mmx %v36 to <4 x i16> + %v38 = bitcast <4 x i16> %v37 to <1 x i64> + br label %if.C + +if.C: + %vx = phi <1 x i64> [ %v21, %if.A ], [ %v38, %if.B ] + %cvt = bitcast <1 x i64> %vx to <2 x i32> + %ex = extractelement <2 x i32> %cvt, i32 0 + %cmp2 = icmp eq i32 %ex, 0 + br i1 %cmp2, label %if.A, label %merge + +merge: +; CHECK: %merge +; CHECK-NOT: movd +; CHECK-NEXT: pshufw + %vy = phi <1 x i64> [ %v21, %if.A ], [ %vx, %if.C ] + %v130 = bitcast <1 x i64> %vy to <4 x i16> + %v131 = bitcast <4 x i16> %v130 to x86_mmx + %v132 = tail call x86_mmx @llvm.x86.sse.pshuf.w(x86_mmx %v131, i8 -18) + %v133 = bitcast x86_mmx %v132 to <4 x i16> + %v134 = bitcast <4 x i16> %v133 to <1 x i64> + %v135 = extractelement <1 x i64> %v134, i32 0 + %v136 = bitcast i64 %v135 to <2 x i32> + %v137 = extractelement <2 x i32> %v136, i32 0 + ret i32 %v137 +} + + +declare x86_mmx @llvm.x86.sse.pshuf.w(x86_mmx, i8) +declare x86_mmx @llvm.x86.mmx.pslli.q(x86_mmx, i32) diff --git a/test/CodeGen/X86/mmx-intrinsics.ll b/test/CodeGen/X86/mmx-intrinsics.ll index 39d481b16e7a..7647fccb5803 100644 --- a/test/CodeGen/X86/mmx-intrinsics.ll +++ b/test/CodeGen/X86/mmx-intrinsics.ll @@ -1,12 +1,13 @@ -; RUN: llc < %s -march=x86 -mattr=+mmx,+ssse3,-avx | FileCheck %s -; RUN: llc < %s -march=x86 -mattr=+avx | FileCheck %s -; RUN: llc < %s -march=x86-64 -mattr=+mmx,+ssse3,-avx | FileCheck %s -; RUN: llc < %s -march=x86-64 -mattr=+avx | FileCheck %s +; RUN: llc < %s -march=x86 -mattr=+mmx,+ssse3,-avx | FileCheck %s --check-prefix=ALL --check-prefix=X86 +; RUN: llc < %s -march=x86 -mattr=+mmx,+avx | FileCheck %s --check-prefix=ALL --check-prefix=X86 +; RUN: llc < %s -march=x86-64 -mattr=+mmx,+ssse3,-avx | FileCheck %s --check-prefix=ALL --check-prefix=X64 +; RUN: llc < %s -march=x86-64 -mattr=+mmx,+avx | FileCheck %s --check-prefix=ALL --check-prefix=X64 declare x86_mmx @llvm.x86.ssse3.phadd.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test1(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: phaddw +; ALL-LABEL: @test1 +; ALL: phaddw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -22,7 +23,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pcmpgt.d(x86_mmx, x86_mmx) nounwind readnone define i64 @test88(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pcmpgtd +; ALL-LABEL: @test88 +; ALL: pcmpgtd entry: %0 = bitcast <1 x i64> %b to <2 x i32> %1 = bitcast <1 x i64> %a to <2 x i32> @@ -38,7 +40,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pcmpgt.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test87(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pcmpgtw +; ALL-LABEL: @test87 +; ALL: pcmpgtw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -54,7 +57,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pcmpgt.b(x86_mmx, x86_mmx) nounwind readnone define i64 @test86(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pcmpgtb +; ALL-LABEL: @test86 +; ALL: pcmpgtb entry: %0 = bitcast <1 x i64> %b to <8 x i8> %1 = bitcast <1 x i64> %a to <8 x i8> @@ -70,7 +74,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pcmpeq.d(x86_mmx, x86_mmx) nounwind readnone define i64 @test85(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pcmpeqd +; ALL-LABEL: @test85 +; ALL: pcmpeqd entry: %0 = bitcast <1 x i64> %b to <2 x i32> %1 = bitcast <1 x i64> %a to <2 x i32> @@ -86,7 +91,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pcmpeq.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test84(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pcmpeqw +; ALL-LABEL: @test84 +; ALL: pcmpeqw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -102,7 +108,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pcmpeq.b(x86_mmx, x86_mmx) nounwind readnone define i64 @test83(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pcmpeqb +; ALL-LABEL: @test83 +; ALL: pcmpeqb entry: %0 = bitcast <1 x i64> %b to <8 x i8> %1 = bitcast <1 x i64> %a to <8 x i8> @@ -118,7 +125,9 @@ entry: declare x86_mmx @llvm.x86.mmx.punpckldq(x86_mmx, x86_mmx) nounwind readnone define i64 @test82(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: punpckldq +; ALL-LABEL: @test82 +; X86: punpckldq {{.*#+}} mm0 = mm0[0],mem[0] +; X64: punpckldq {{.*#+}} mm0 = mm0[0],mm1[0] entry: %0 = bitcast <1 x i64> %b to <2 x i32> %1 = bitcast <1 x i64> %a to <2 x i32> @@ -134,7 +143,9 @@ entry: declare x86_mmx @llvm.x86.mmx.punpcklwd(x86_mmx, x86_mmx) nounwind readnone define i64 @test81(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: punpcklwd +; ALL-LABEL: @test81 +; X86: punpcklwd {{.*#+}} mm0 = mm0[0],mem[0],mm0[1],mem[1] +; X64: punpcklwd {{.*#+}} mm0 = mm0[0],mm1[0],mm0[1],mm1[1] entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -150,7 +161,9 @@ entry: declare x86_mmx @llvm.x86.mmx.punpcklbw(x86_mmx, x86_mmx) nounwind readnone define i64 @test80(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: punpcklbw +; ALL-LABEL: @test80 +; X86: punpcklbw {{.*#+}} mm0 = mm0[0],mem[0],mm0[1],mem[1],mm0[2],mem[2],mm0[3],mem[3] +; X64: punpcklbw {{.*#+}} mm0 = mm0[0],mm1[0],mm0[1],mm1[1],mm0[2],mm1[2],mm0[3],mm1[3] entry: %0 = bitcast <1 x i64> %b to <8 x i8> %1 = bitcast <1 x i64> %a to <8 x i8> @@ -166,7 +179,9 @@ entry: declare x86_mmx @llvm.x86.mmx.punpckhdq(x86_mmx, x86_mmx) nounwind readnone define i64 @test79(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: punpckhdq +; ALL-LABEL: @test79 +; X86: punpckhdq {{.*#+}} mm0 = mm0[1],mem[1] +; X64: punpckhdq {{.*#+}} mm0 = mm0[1],mm1[1] entry: %0 = bitcast <1 x i64> %b to <2 x i32> %1 = bitcast <1 x i64> %a to <2 x i32> @@ -182,7 +197,9 @@ entry: declare x86_mmx @llvm.x86.mmx.punpckhwd(x86_mmx, x86_mmx) nounwind readnone define i64 @test78(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: punpckhwd +; ALL-LABEL: @test78 +; X86: punpckhwd {{.*#+}} mm0 = mm0[2],mem[2],mm0[3],mem[3] +; X64: punpckhwd {{.*#+}} mm0 = mm0[2],mm1[2],mm0[3],mm1[3] entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -198,7 +215,9 @@ entry: declare x86_mmx @llvm.x86.mmx.punpckhbw(x86_mmx, x86_mmx) nounwind readnone define i64 @test77(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: punpckhbw +; ALL-LABEL: @test77 +; X86: punpckhbw {{.*#+}} mm0 = mm0[4],mem[4],mm0[5],mem[5],mm0[6],mem[6],mm0[7],mem[7] +; X64: punpckhbw {{.*#+}} mm0 = mm0[4],mm1[4],mm0[5],mm1[5],mm0[6],mm1[6],mm0[7],mm1[7] entry: %0 = bitcast <1 x i64> %b to <8 x i8> %1 = bitcast <1 x i64> %a to <8 x i8> @@ -214,7 +233,8 @@ entry: declare x86_mmx @llvm.x86.mmx.packuswb(x86_mmx, x86_mmx) nounwind readnone define i64 @test76(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: packuswb +; ALL-LABEL: @test76 +; ALL: packuswb entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -230,7 +250,8 @@ entry: declare x86_mmx @llvm.x86.mmx.packssdw(x86_mmx, x86_mmx) nounwind readnone define i64 @test75(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: packssdw +; ALL-LABEL: @test75 +; ALL: packssdw entry: %0 = bitcast <1 x i64> %b to <2 x i32> %1 = bitcast <1 x i64> %a to <2 x i32> @@ -246,7 +267,8 @@ entry: declare x86_mmx @llvm.x86.mmx.packsswb(x86_mmx, x86_mmx) nounwind readnone define i64 @test74(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: packsswb +; ALL-LABEL: @test74 +; ALL: packsswb entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -262,7 +284,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psrai.d(x86_mmx, i32) nounwind readnone define i64 @test73(<1 x i64> %a) nounwind readnone optsize ssp { -; CHECK: psrad +; ALL-LABEL: @test73 +; ALL: psrad entry: %0 = bitcast <1 x i64> %a to <2 x i32> %mmx_var.i = bitcast <2 x i32> %0 to x86_mmx @@ -276,7 +299,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psrai.w(x86_mmx, i32) nounwind readnone define i64 @test72(<1 x i64> %a) nounwind readnone optsize ssp { -; CHECK: psraw +; ALL-LABEL: @test72 +; ALL: psraw entry: %0 = bitcast <1 x i64> %a to <4 x i16> %mmx_var.i = bitcast <4 x i16> %0 to x86_mmx @@ -290,7 +314,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psrli.q(x86_mmx, i32) nounwind readnone define i64 @test71(<1 x i64> %a) nounwind readnone optsize ssp { -; CHECK: psrlq +; ALL-LABEL: @test71 +; ALL: psrlq entry: %0 = extractelement <1 x i64> %a, i32 0 %mmx_var.i = bitcast i64 %0 to x86_mmx @@ -302,7 +327,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psrli.d(x86_mmx, i32) nounwind readnone define i64 @test70(<1 x i64> %a) nounwind readnone optsize ssp { -; CHECK: psrld +; ALL-LABEL: @test70 +; ALL: psrld entry: %0 = bitcast <1 x i64> %a to <2 x i32> %mmx_var.i = bitcast <2 x i32> %0 to x86_mmx @@ -316,7 +342,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psrli.w(x86_mmx, i32) nounwind readnone define i64 @test69(<1 x i64> %a) nounwind readnone optsize ssp { -; CHECK: psrlw +; ALL-LABEL: @test69 +; ALL: psrlw entry: %0 = bitcast <1 x i64> %a to <4 x i16> %mmx_var.i = bitcast <4 x i16> %0 to x86_mmx @@ -330,7 +357,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pslli.q(x86_mmx, i32) nounwind readnone define i64 @test68(<1 x i64> %a) nounwind readnone optsize ssp { -; CHECK: psllq +; ALL-LABEL: @test68 +; ALL: psllq entry: %0 = extractelement <1 x i64> %a, i32 0 %mmx_var.i = bitcast i64 %0 to x86_mmx @@ -342,7 +370,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pslli.d(x86_mmx, i32) nounwind readnone define i64 @test67(<1 x i64> %a) nounwind readnone optsize ssp { -; CHECK: pslld +; ALL-LABEL: @test67 +; ALL: pslld entry: %0 = bitcast <1 x i64> %a to <2 x i32> %mmx_var.i = bitcast <2 x i32> %0 to x86_mmx @@ -356,7 +385,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pslli.w(x86_mmx, i32) nounwind readnone define i64 @test66(<1 x i64> %a) nounwind readnone optsize ssp { -; CHECK: psllw +; ALL-LABEL: @test66 +; ALL: psllw entry: %0 = bitcast <1 x i64> %a to <4 x i16> %mmx_var.i = bitcast <4 x i16> %0 to x86_mmx @@ -370,7 +400,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psra.d(x86_mmx, x86_mmx) nounwind readnone define i64 @test65(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psrad +; ALL-LABEL: @test65 +; ALL: psrad entry: %0 = bitcast <1 x i64> %a to <2 x i32> %mmx_var.i = bitcast <2 x i32> %0 to x86_mmx @@ -386,7 +417,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psra.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test64(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psraw +; ALL-LABEL: @test64 +; ALL: psraw entry: %0 = bitcast <1 x i64> %a to <4 x i16> %mmx_var.i = bitcast <4 x i16> %0 to x86_mmx @@ -402,7 +434,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psrl.q(x86_mmx, x86_mmx) nounwind readnone define i64 @test63(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psrlq +; ALL-LABEL: @test63 +; ALL: psrlq entry: %0 = extractelement <1 x i64> %a, i32 0 %mmx_var.i = bitcast i64 %0 to x86_mmx @@ -416,7 +449,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psrl.d(x86_mmx, x86_mmx) nounwind readnone define i64 @test62(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psrld +; ALL-LABEL: @test62 +; ALL: psrld entry: %0 = bitcast <1 x i64> %a to <2 x i32> %mmx_var.i = bitcast <2 x i32> %0 to x86_mmx @@ -432,7 +466,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psrl.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test61(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psrlw +; ALL-LABEL: @test61 +; ALL: psrlw entry: %0 = bitcast <1 x i64> %a to <4 x i16> %mmx_var.i = bitcast <4 x i16> %0 to x86_mmx @@ -448,7 +483,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psll.q(x86_mmx, x86_mmx) nounwind readnone define i64 @test60(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psllq +; ALL-LABEL: @test60 +; ALL: psllq entry: %0 = extractelement <1 x i64> %a, i32 0 %mmx_var.i = bitcast i64 %0 to x86_mmx @@ -462,7 +498,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psll.d(x86_mmx, x86_mmx) nounwind readnone define i64 @test59(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pslld +; ALL-LABEL: @test59 +; ALL: pslld entry: %0 = bitcast <1 x i64> %a to <2 x i32> %mmx_var.i = bitcast <2 x i32> %0 to x86_mmx @@ -478,7 +515,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psll.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test58(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psllw +; ALL-LABEL: @test58 +; ALL: psllw entry: %0 = bitcast <1 x i64> %a to <4 x i16> %mmx_var.i = bitcast <4 x i16> %0 to x86_mmx @@ -494,7 +532,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pxor(x86_mmx, x86_mmx) nounwind readnone define i64 @test56(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pxor +; ALL-LABEL: @test56 +; ALL: pxor entry: %0 = bitcast <1 x i64> %b to <2 x i32> %1 = bitcast <1 x i64> %a to <2 x i32> @@ -510,7 +549,8 @@ entry: declare x86_mmx @llvm.x86.mmx.por(x86_mmx, x86_mmx) nounwind readnone define i64 @test55(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: por +; ALL-LABEL: @test55 +; ALL: por entry: %0 = bitcast <1 x i64> %b to <2 x i32> %1 = bitcast <1 x i64> %a to <2 x i32> @@ -526,7 +566,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pandn(x86_mmx, x86_mmx) nounwind readnone define i64 @test54(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pandn +; ALL-LABEL: @test54 +; ALL: pandn entry: %0 = bitcast <1 x i64> %b to <2 x i32> %1 = bitcast <1 x i64> %a to <2 x i32> @@ -542,7 +583,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pand(x86_mmx, x86_mmx) nounwind readnone define i64 @test53(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pand +; ALL-LABEL: @test53 +; ALL: pand entry: %0 = bitcast <1 x i64> %b to <2 x i32> %1 = bitcast <1 x i64> %a to <2 x i32> @@ -558,7 +600,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pmull.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test52(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pmullw +; ALL-LABEL: @test52 +; ALL: pmullw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -572,7 +615,8 @@ entry: } define i64 @test51(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pmullw +; ALL-LABEL: @test51 +; ALL: pmullw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -588,7 +632,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pmulh.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test50(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pmulhw +; ALL-LABEL: @test50 +; ALL: pmulhw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -604,7 +649,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pmadd.wd(x86_mmx, x86_mmx) nounwind readnone define i64 @test49(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pmaddwd +; ALL-LABEL: @test49 +; ALL: pmaddwd entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -620,7 +666,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psubus.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test48(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psubusw +; ALL-LABEL: @test48 +; ALL: psubusw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -636,7 +683,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psubus.b(x86_mmx, x86_mmx) nounwind readnone define i64 @test47(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psubusb +; ALL-LABEL: @test47 +; ALL: psubusb entry: %0 = bitcast <1 x i64> %b to <8 x i8> %1 = bitcast <1 x i64> %a to <8 x i8> @@ -652,7 +700,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psubs.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test46(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psubsw +; ALL-LABEL: @test46 +; ALL: psubsw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -668,7 +717,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psubs.b(x86_mmx, x86_mmx) nounwind readnone define i64 @test45(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psubsb +; ALL-LABEL: @test45 +; ALL: psubsb entry: %0 = bitcast <1 x i64> %b to <8 x i8> %1 = bitcast <1 x i64> %a to <8 x i8> @@ -682,7 +732,8 @@ entry: } define i64 @test44(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psubq +; ALL-LABEL: @test44 +; ALL: psubq entry: %0 = extractelement <1 x i64> %a, i32 0 %mmx_var = bitcast i64 %0 to x86_mmx @@ -698,7 +749,8 @@ declare x86_mmx @llvm.x86.mmx.psub.q(x86_mmx, x86_mmx) nounwind readnone declare x86_mmx @llvm.x86.mmx.psub.d(x86_mmx, x86_mmx) nounwind readnone define i64 @test43(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psubd +; ALL-LABEL: @test43 +; ALL: psubd entry: %0 = bitcast <1 x i64> %b to <2 x i32> %1 = bitcast <1 x i64> %a to <2 x i32> @@ -714,7 +766,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psub.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test42(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psubw +; ALL-LABEL: @test42 +; ALL: psubw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -730,7 +783,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psub.b(x86_mmx, x86_mmx) nounwind readnone define i64 @test41(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psubb +; ALL-LABEL: @test41 +; ALL: psubb entry: %0 = bitcast <1 x i64> %b to <8 x i8> %1 = bitcast <1 x i64> %a to <8 x i8> @@ -746,7 +800,8 @@ entry: declare x86_mmx @llvm.x86.mmx.paddus.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test40(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: paddusw +; ALL-LABEL: @test40 +; ALL: paddusw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -762,7 +817,8 @@ entry: declare x86_mmx @llvm.x86.mmx.paddus.b(x86_mmx, x86_mmx) nounwind readnone define i64 @test39(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: paddusb +; ALL-LABEL: @test39 +; ALL: paddusb entry: %0 = bitcast <1 x i64> %b to <8 x i8> %1 = bitcast <1 x i64> %a to <8 x i8> @@ -778,7 +834,8 @@ entry: declare x86_mmx @llvm.x86.mmx.padds.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test38(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: paddsw +; ALL-LABEL: @test38 +; ALL: paddsw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -794,7 +851,8 @@ entry: declare x86_mmx @llvm.x86.mmx.padds.b(x86_mmx, x86_mmx) nounwind readnone define i64 @test37(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: paddsb +; ALL-LABEL: @test37 +; ALL: paddsb entry: %0 = bitcast <1 x i64> %b to <8 x i8> %1 = bitcast <1 x i64> %a to <8 x i8> @@ -810,7 +868,8 @@ entry: declare x86_mmx @llvm.x86.mmx.padd.q(x86_mmx, x86_mmx) nounwind readnone define i64 @test36(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: paddq +; ALL-LABEL: @test36 +; ALL: paddq entry: %0 = extractelement <1 x i64> %a, i32 0 %mmx_var = bitcast i64 %0 to x86_mmx @@ -824,7 +883,8 @@ entry: declare x86_mmx @llvm.x86.mmx.padd.d(x86_mmx, x86_mmx) nounwind readnone define i64 @test35(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: paddd +; ALL-LABEL: @test35 +; ALL: paddd entry: %0 = bitcast <1 x i64> %b to <2 x i32> %1 = bitcast <1 x i64> %a to <2 x i32> @@ -840,7 +900,8 @@ entry: declare x86_mmx @llvm.x86.mmx.padd.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test34(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: paddw +; ALL-LABEL: @test34 +; ALL: paddw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -856,7 +917,8 @@ entry: declare x86_mmx @llvm.x86.mmx.padd.b(x86_mmx, x86_mmx) nounwind readnone define i64 @test33(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: paddb +; ALL-LABEL: @test33 +; ALL: paddb entry: %0 = bitcast <1 x i64> %b to <8 x i8> %1 = bitcast <1 x i64> %a to <8 x i8> @@ -872,7 +934,8 @@ entry: declare x86_mmx @llvm.x86.mmx.psad.bw(x86_mmx, x86_mmx) nounwind readnone define i64 @test32(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psadbw +; ALL-LABEL: @test32 +; ALL: psadbw entry: %0 = bitcast <1 x i64> %b to <8 x i8> %1 = bitcast <1 x i64> %a to <8 x i8> @@ -886,7 +949,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pmins.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test31(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pminsw +; ALL-LABEL: @test31 +; ALL: pminsw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -902,7 +966,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pminu.b(x86_mmx, x86_mmx) nounwind readnone define i64 @test30(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pminub +; ALL-LABEL: @test30 +; ALL: pminub entry: %0 = bitcast <1 x i64> %b to <8 x i8> %1 = bitcast <1 x i64> %a to <8 x i8> @@ -918,7 +983,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pmaxs.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test29(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pmaxsw +; ALL-LABEL: @test29 +; ALL: pmaxsw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -934,7 +1000,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pmaxu.b(x86_mmx, x86_mmx) nounwind readnone define i64 @test28(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pmaxub +; ALL-LABEL: @test28 +; ALL: pmaxub entry: %0 = bitcast <1 x i64> %b to <8 x i8> %1 = bitcast <1 x i64> %a to <8 x i8> @@ -950,7 +1017,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pavg.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test27(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pavgw +; ALL-LABEL: @test27 +; ALL: pavgw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -966,7 +1034,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pavg.b(x86_mmx, x86_mmx) nounwind readnone define i64 @test26(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pavgb +; ALL-LABEL: @test26 +; ALL: pavgb entry: %0 = bitcast <1 x i64> %b to <8 x i8> %1 = bitcast <1 x i64> %a to <8 x i8> @@ -982,7 +1051,8 @@ entry: declare void @llvm.x86.mmx.movnt.dq(x86_mmx*, x86_mmx) nounwind define void @test25(<1 x i64>* %p, <1 x i64> %a) nounwind optsize ssp { -; CHECK: movntq +; ALL-LABEL: @test25 +; ALL: movntq entry: %mmx_ptr_var.i = bitcast <1 x i64>* %p to x86_mmx* %0 = extractelement <1 x i64> %a, i32 0 @@ -994,7 +1064,8 @@ entry: declare i32 @llvm.x86.mmx.pmovmskb(x86_mmx) nounwind readnone define i32 @test24(<1 x i64> %a) nounwind readnone optsize ssp { -; CHECK: pmovmskb +; ALL-LABEL: @test24 +; ALL: pmovmskb entry: %0 = bitcast <1 x i64> %a to <8 x i8> %mmx_var.i = bitcast <8 x i8> %0 to x86_mmx @@ -1005,7 +1076,8 @@ entry: declare void @llvm.x86.mmx.maskmovq(x86_mmx, x86_mmx, i8*) nounwind define void @test23(<1 x i64> %d, <1 x i64> %n, i8* %p) nounwind optsize ssp { -; CHECK: maskmovq +; ALL-LABEL: @test23 +; ALL: maskmovq entry: %0 = bitcast <1 x i64> %n to <8 x i8> %1 = bitcast <1 x i64> %d to <8 x i8> @@ -1018,7 +1090,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pmulhu.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test22(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pmulhuw +; ALL-LABEL: @test22 +; ALL: pmulhuw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -1034,7 +1107,9 @@ entry: declare x86_mmx @llvm.x86.sse.pshuf.w(x86_mmx, i8) nounwind readnone define i64 @test21(<1 x i64> %a) nounwind readnone optsize ssp { -; CHECK: pshufw +; ALL-LABEL: @test21 +; X86: pshufw {{.*#+}} mm0 = mem[3,0,0,0] +; X64: pshufw {{.*#+}} mm0 = mm0[3,0,0,0] entry: %0 = bitcast <1 x i64> %a to <4 x i16> %1 = bitcast <4 x i16> %0 to x86_mmx @@ -1046,9 +1121,10 @@ entry: } define i32 @test21_2(<1 x i64> %a) nounwind readnone optsize ssp { -; CHECK: test21_2 -; CHECK: pshufw -; CHECK: movd +; ALL-LABEL: @test21_2 +; X86: pshufw {{.*#+}} mm0 = mem[3,0,0,0] +; X64: pshufw {{.*#+}} mm0 = mm0[3,0,0,0] +; ALL: movd entry: %0 = bitcast <1 x i64> %a to <4 x i16> %1 = bitcast <4 x i16> %0 to x86_mmx @@ -1062,7 +1138,8 @@ entry: declare x86_mmx @llvm.x86.mmx.pmulu.dq(x86_mmx, x86_mmx) nounwind readnone define i64 @test20(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pmuludq +; ALL-LABEL: @test20 +; ALL: pmuludq entry: %0 = bitcast <1 x i64> %b to <2 x i32> %1 = bitcast <1 x i64> %a to <2 x i32> @@ -1076,7 +1153,8 @@ entry: declare <2 x double> @llvm.x86.sse.cvtpi2pd(x86_mmx) nounwind readnone define <2 x double> @test19(<1 x i64> %a) nounwind readnone optsize ssp { -; CHECK: cvtpi2pd +; ALL-LABEL: @test19 +; ALL: cvtpi2pd entry: %0 = bitcast <1 x i64> %a to <2 x i32> %1 = bitcast <2 x i32> %0 to x86_mmx @@ -1087,7 +1165,8 @@ entry: declare x86_mmx @llvm.x86.sse.cvttpd2pi(<2 x double>) nounwind readnone define i64 @test18(<2 x double> %a) nounwind readnone optsize ssp { -; CHECK: cvttpd2pi +; ALL-LABEL: @test18 +; ALL: cvttpd2pi entry: %0 = tail call x86_mmx @llvm.x86.sse.cvttpd2pi(<2 x double> %a) nounwind readnone %1 = bitcast x86_mmx %0 to <2 x i32> @@ -1099,7 +1178,8 @@ entry: declare x86_mmx @llvm.x86.sse.cvtpd2pi(<2 x double>) nounwind readnone define i64 @test17(<2 x double> %a) nounwind readnone optsize ssp { -; CHECK: cvtpd2pi +; ALL-LABEL: @test17 +; ALL: cvtpd2pi entry: %0 = tail call x86_mmx @llvm.x86.sse.cvtpd2pi(<2 x double> %a) nounwind readnone %1 = bitcast x86_mmx %0 to <2 x i32> @@ -1111,7 +1191,8 @@ entry: declare x86_mmx @llvm.x86.mmx.palignr.b(x86_mmx, x86_mmx, i8) nounwind readnone define i64 @test16(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: palignr +; ALL-LABEL: @test16 +; ALL: palignr entry: %0 = extractelement <1 x i64> %a, i32 0 %mmx_var = bitcast i64 %0 to x86_mmx @@ -1125,7 +1206,8 @@ entry: declare x86_mmx @llvm.x86.ssse3.pabs.d(x86_mmx) nounwind readnone define i64 @test15(<1 x i64> %a) nounwind readnone optsize ssp { -; CHECK: pabsd +; ALL-LABEL: @test15 +; ALL: pabsd entry: %0 = bitcast <1 x i64> %a to <2 x i32> %1 = bitcast <2 x i32> %0 to x86_mmx @@ -1139,7 +1221,8 @@ entry: declare x86_mmx @llvm.x86.ssse3.pabs.w(x86_mmx) nounwind readnone define i64 @test14(<1 x i64> %a) nounwind readnone optsize ssp { -; CHECK: pabsw +; ALL-LABEL: @test14 +; ALL: pabsw entry: %0 = bitcast <1 x i64> %a to <4 x i16> %1 = bitcast <4 x i16> %0 to x86_mmx @@ -1153,7 +1236,8 @@ entry: declare x86_mmx @llvm.x86.ssse3.pabs.b(x86_mmx) nounwind readnone define i64 @test13(<1 x i64> %a) nounwind readnone optsize ssp { -; CHECK: pabsb +; ALL-LABEL: @test13 +; ALL: pabsb entry: %0 = bitcast <1 x i64> %a to <8 x i8> %1 = bitcast <8 x i8> %0 to x86_mmx @@ -1167,7 +1251,8 @@ entry: declare x86_mmx @llvm.x86.ssse3.psign.d(x86_mmx, x86_mmx) nounwind readnone define i64 @test12(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psignd +; ALL-LABEL: @test12 +; ALL: psignd entry: %0 = bitcast <1 x i64> %b to <2 x i32> %1 = bitcast <1 x i64> %a to <2 x i32> @@ -1183,7 +1268,8 @@ entry: declare x86_mmx @llvm.x86.ssse3.psign.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test11(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psignw +; ALL-LABEL: @test11 +; ALL: psignw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -1199,7 +1285,8 @@ entry: declare x86_mmx @llvm.x86.ssse3.psign.b(x86_mmx, x86_mmx) nounwind readnone define i64 @test10(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: psignb +; ALL-LABEL: @test10 +; ALL: psignb entry: %0 = bitcast <1 x i64> %b to <8 x i8> %1 = bitcast <1 x i64> %a to <8 x i8> @@ -1215,7 +1302,8 @@ entry: declare x86_mmx @llvm.x86.ssse3.pshuf.b(x86_mmx, x86_mmx) nounwind readnone define i64 @test9(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pshufb +; ALL-LABEL: @test9 +; ALL: pshufb entry: %0 = bitcast <1 x i64> %b to <8 x i8> %1 = bitcast <1 x i64> %a to <8 x i8> @@ -1231,7 +1319,8 @@ entry: declare x86_mmx @llvm.x86.ssse3.pmul.hr.sw(x86_mmx, x86_mmx) nounwind readnone define i64 @test8(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pmulhrsw +; ALL-LABEL: @test8 +; ALL: pmulhrsw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -1247,7 +1336,8 @@ entry: declare x86_mmx @llvm.x86.ssse3.pmadd.ub.sw(x86_mmx, x86_mmx) nounwind readnone define i64 @test7(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: pmaddubsw +; ALL-LABEL: @test7 +; ALL: pmaddubsw entry: %0 = bitcast <1 x i64> %b to <8 x i8> %1 = bitcast <1 x i64> %a to <8 x i8> @@ -1263,7 +1353,8 @@ entry: declare x86_mmx @llvm.x86.ssse3.phsub.sw(x86_mmx, x86_mmx) nounwind readnone define i64 @test6(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: phsubsw +; ALL-LABEL: @test6 +; ALL: phsubsw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -1279,7 +1370,8 @@ entry: declare x86_mmx @llvm.x86.ssse3.phsub.d(x86_mmx, x86_mmx) nounwind readnone define i64 @test5(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: phsubd +; ALL-LABEL: @test5 +; ALL: phsubd entry: %0 = bitcast <1 x i64> %b to <2 x i32> %1 = bitcast <1 x i64> %a to <2 x i32> @@ -1295,7 +1387,8 @@ entry: declare x86_mmx @llvm.x86.ssse3.phsub.w(x86_mmx, x86_mmx) nounwind readnone define i64 @test4(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: phsubw +; ALL-LABEL: @test4 +; ALL: phsubw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -1311,7 +1404,8 @@ entry: declare x86_mmx @llvm.x86.ssse3.phadd.sw(x86_mmx, x86_mmx) nounwind readnone define i64 @test3(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: phaddsw +; ALL-LABEL: @test3 +; ALL: phaddsw entry: %0 = bitcast <1 x i64> %b to <4 x i16> %1 = bitcast <1 x i64> %a to <4 x i16> @@ -1327,7 +1421,8 @@ entry: declare x86_mmx @llvm.x86.ssse3.phadd.d(x86_mmx, x86_mmx) nounwind readnone define i64 @test2(<1 x i64> %a, <1 x i64> %b) nounwind readnone optsize ssp { -; CHECK: phaddd +; ALL-LABEL: @test2 +; ALL: phaddd entry: %0 = bitcast <1 x i64> %b to <2 x i32> %1 = bitcast <1 x i64> %a to <2 x i32> @@ -1341,16 +1436,18 @@ entry: } define <4 x float> @test89(<4 x float> %a, x86_mmx %b) nounwind { -; CHECK: cvtpi2ps +; ALL-LABEL: @test89 +; ALL: cvtpi2ps %c = tail call <4 x float> @llvm.x86.sse.cvtpi2ps(<4 x float> %a, x86_mmx %b) ret <4 x float> %c } declare <4 x float> @llvm.x86.sse.cvtpi2ps(<4 x float>, x86_mmx) nounwind readnone -; CHECK-LABEL: test90 +; ALL-LABEL: test90 define void @test90() { -; CHECK: emms +; ALL-LABEL: @test90 +; ALL: emms call void @llvm.x86.mmx.emms() ret void } diff --git a/test/CodeGen/X86/mmx-only.ll b/test/CodeGen/X86/mmx-only.ll new file mode 100644 index 000000000000..35598d5f6e19 --- /dev/null +++ b/test/CodeGen/X86/mmx-only.ll @@ -0,0 +1,21 @@ +; RUN: llc < %s -march=x86 -mattr=+mmx | FileCheck %s +; RUN: llc < %s -march=x86 -mattr=+mmx,-sse | FileCheck %s + +; Test that turning off sse doesn't turn off mmx. + +declare x86_mmx @llvm.x86.mmx.pcmpgt.d(x86_mmx, x86_mmx) nounwind readnone + +define i64 @test88(<1 x i64> %a, <1 x i64> %b) nounwind readnone { +; CHECK-LABEL: @test88 +; CHECK: pcmpgtd +entry: + %0 = bitcast <1 x i64> %b to <2 x i32> + %1 = bitcast <1 x i64> %a to <2 x i32> + %mmx_var.i = bitcast <2 x i32> %1 to x86_mmx + %mmx_var1.i = bitcast <2 x i32> %0 to x86_mmx + %2 = tail call x86_mmx @llvm.x86.mmx.pcmpgt.d(x86_mmx %mmx_var.i, x86_mmx %mmx_var1.i) nounwind + %3 = bitcast x86_mmx %2 to <2 x i32> + %4 = bitcast <2 x i32> %3 to <1 x i64> + %5 = extractelement <1 x i64> %4, i32 0 + ret i64 %5 +} diff --git a/test/CodeGen/X86/movntdq-no-avx.ll b/test/CodeGen/X86/movntdq-no-avx.ll index cc35e201e6b3..2bf09dd6f581 100644 --- a/test/CodeGen/X86/movntdq-no-avx.ll +++ b/test/CodeGen/X86/movntdq-no-avx.ll @@ -5,7 +5,7 @@ define void @test(<2 x i64>* nocapture %a, <2 x i64> %b) nounwind optsize { entry: - store <2 x i64> %b, <2 x i64>* %a, align 16, !nontemporal !0 + store <2 x i64> %b, <2 x i64>* %a, align 32, !nontemporal !0 ret void } diff --git a/test/CodeGen/X86/movpc32-check.ll b/test/CodeGen/X86/movpc32-check.ll new file mode 100644 index 000000000000..606af3c898f4 --- /dev/null +++ b/test/CodeGen/X86/movpc32-check.ll @@ -0,0 +1,42 @@ +; RUN: llc < %s -mtriple=i686-pc-linux -relocation-model=pic | FileCheck %s + +target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128" +target triple = "i686-pc-linux" + +; Function Attrs: nounwind +define void @test() #0 !dbg !4 { +entry: + call void bitcast (void (...)* @bar to void ()*)(), !dbg !11 + ret void, !dbg !12 +} + +declare void @bar(...) #1 + +attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="i686" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="i686" "unsafe-fp-math"="false" "use-soft-float"="false" } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!7, !8, !9} +!llvm.ident = !{!10} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0 (http://llvm.org/git/clang.git 3490ab8630d5643f71f1f04e46984f05b27b8d67) (http://llvm.org/git/llvm.git d2643e2ff955ed234944fe3c6b4ffc1250085843)", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2, subprograms: !3) +!1 = !DIFile(filename: "test.c", directory: "movpc-test") +!2 = !{} +!3 = !{!4} +!4 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: false, variables: !2) +!5 = !DISubroutineType(types: !6) +!6 = !{null} +!7 = !{i32 2, !"Dwarf Version", i32 4} +!8 = !{i32 2, !"Debug Info Version", i32 3} +!9 = !{i32 1, !"PIC Level", i32 2} +!10 = !{!"clang version 3.8.0 (http://llvm.org/git/clang.git 3490ab8630d5643f71f1f04e46984f05b27b8d67) (http://llvm.org/git/llvm.git d2643e2ff955ed234944fe3c6b4ffc1250085843)"} +!11 = !DILocation(line: 4, column: 3, scope: !4) +!12 = !DILocation(line: 5, column: 1, scope: !4) + +; CHECK: calll .L0$pb +; CHECK-NEXT: .Ltmp3: +; CHECK-NEXT: .cfi_adjust_cfa_offset 4 +; CHECK-NEXT: .L0$pb: +; CHECK-NEXT: popl +; CHECK-NEXT: .Ltmp4: +; CHECK-NEXT: .cfi_adjust_cfa_offset -4 diff --git a/test/CodeGen/X86/movtopush.ll b/test/CodeGen/X86/movtopush.ll index b02f9ec45e7f..de4c87cf30ad 100644 --- a/test/CodeGen/X86/movtopush.ll +++ b/test/CodeGen/X86/movtopush.ll @@ -1,6 +1,6 @@ ; RUN: llc < %s -mtriple=i686-windows | FileCheck %s -check-prefix=NORMAL ; RUN: llc < %s -mtriple=x86_64-windows | FileCheck %s -check-prefix=X64 -; RUN: llc < %s -mtriple=i686-windows -force-align-stack -stack-alignment=32 | FileCheck %s -check-prefix=ALIGNED +; RUN: llc < %s -mtriple=i686-windows -stackrealign -stack-alignment=32 | FileCheck %s -check-prefix=ALIGNED %class.Class = type { i32 } %struct.s = type { i64 } @@ -357,3 +357,26 @@ entry: call void @good(i32 9, i32 10, i32 11, i32 12) ret void } + +; Make sure the add does not prevent folding loads into pushes. +; val1 and val2 will not be folded into pushes since they have +; an additional use, but val3 should be. +; NORMAL-LABEL: test13: +; NORMAL: movl ([[P1:%e..]]), [[V1:%e..]] +; NORMAL-NEXT: movl ([[P2:%e..]]), [[V2:%e..]] +; NORMAL-NEXT: , [[ADD:%e..]] +; NORMAL-NEXT: pushl [[ADD]] +; NORMAL-NEXT: pushl ([[P3:%e..]]) +; NORMAL-NEXT: pushl [[V2]] +; NORMAL-NEXT: pushl [[V1]] +; NORMAL-NEXT: calll _good +; NORMAL: movl [[P3]], %eax +define i32* @test13(i32* inreg %ptr1, i32* inreg %ptr2, i32* inreg %ptr3) optsize { +entry: + %val1 = load i32, i32* %ptr1 + %val2 = load i32, i32* %ptr2 + %val3 = load i32, i32* %ptr3 + %add = add i32 %val1, %val2 + call void @good(i32 %val1, i32 %val2, i32 %val3, i32 %add) + ret i32* %ptr3 +} diff --git a/test/CodeGen/X86/mult-alt-x86.ll b/test/CodeGen/X86/mult-alt-x86.ll index 5174f85adb9f..1c83fedad3ce 100644 --- a/test/CodeGen/X86/mult-alt-x86.ll +++ b/test/CodeGen/X86/mult-alt-x86.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -march=x86 -mattr=+sse2 -no-integrated-as +; RUN: llc < %s -march=x86 -mattr=+mmx,+sse2 -no-integrated-as ; ModuleID = 'mult-alt-x86.c' target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f80:128:128-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32" target triple = "i686-pc-win32" diff --git a/test/CodeGen/X86/musttail-varargs.ll b/test/CodeGen/X86/musttail-varargs.ll index 3613f4c08cce..247d78776b80 100644 --- a/test/CodeGen/X86/musttail-varargs.ll +++ b/test/CodeGen/X86/musttail-varargs.ll @@ -1,4 +1,5 @@ ; RUN: llc < %s -enable-tail-merge=0 -mtriple=x86_64-linux | FileCheck %s --check-prefix=LINUX +; RUN: llc < %s -enable-tail-merge=0 -mtriple=x86_64-linux-gnux32 | FileCheck %s --check-prefix=LINUX-X32 ; RUN: llc < %s -enable-tail-merge=0 -mtriple=x86_64-windows | FileCheck %s --check-prefix=WINDOWS ; RUN: llc < %s -enable-tail-merge=0 -mtriple=i686-windows | FileCheck %s --check-prefix=X86 @@ -57,6 +58,40 @@ define void @f_thunk(i8* %this, ...) { ; LINUX-DAG: movb {{.*}}, %al ; LINUX: jmpq *{{.*}} # TAILCALL +; LINUX-X32-LABEL: f_thunk: +; LINUX-X32-DAG: movl %edi, {{.*}} +; LINUX-X32-DAG: movq %rsi, {{.*}} +; LINUX-X32-DAG: movq %rdx, {{.*}} +; LINUX-X32-DAG: movq %rcx, {{.*}} +; LINUX-X32-DAG: movq %r8, {{.*}} +; LINUX-X32-DAG: movq %r9, {{.*}} +; LINUX-X32-DAG: movb %al, {{.*}} +; LINUX-X32-DAG: movaps %xmm0, {{[0-9]*}}(%esp) +; LINUX-X32-DAG: movaps %xmm1, {{[0-9]*}}(%esp) +; LINUX-X32-DAG: movaps %xmm2, {{[0-9]*}}(%esp) +; LINUX-X32-DAG: movaps %xmm3, {{[0-9]*}}(%esp) +; LINUX-X32-DAG: movaps %xmm4, {{[0-9]*}}(%esp) +; LINUX-X32-DAG: movaps %xmm5, {{[0-9]*}}(%esp) +; LINUX-X32-DAG: movaps %xmm6, {{[0-9]*}}(%esp) +; LINUX-X32-DAG: movaps %xmm7, {{[0-9]*}}(%esp) +; LINUX-X32: callq get_f +; LINUX-X32-DAG: movaps {{[0-9]*}}(%esp), %xmm0 +; LINUX-X32-DAG: movaps {{[0-9]*}}(%esp), %xmm1 +; LINUX-X32-DAG: movaps {{[0-9]*}}(%esp), %xmm2 +; LINUX-X32-DAG: movaps {{[0-9]*}}(%esp), %xmm3 +; LINUX-X32-DAG: movaps {{[0-9]*}}(%esp), %xmm4 +; LINUX-X32-DAG: movaps {{[0-9]*}}(%esp), %xmm5 +; LINUX-X32-DAG: movaps {{[0-9]*}}(%esp), %xmm6 +; LINUX-X32-DAG: movaps {{[0-9]*}}(%esp), %xmm7 +; LINUX-X32-DAG: movl {{.*}}, %edi +; LINUX-X32-DAG: movq {{.*}}, %rsi +; LINUX-X32-DAG: movq {{.*}}, %rdx +; LINUX-X32-DAG: movq {{.*}}, %rcx +; LINUX-X32-DAG: movq {{.*}}, %r8 +; LINUX-X32-DAG: movq {{.*}}, %r9 +; LINUX-X32-DAG: movb {{.*}}, %al +; LINUX-X32: jmpq *{{.*}} # TAILCALL + ; WINDOWS-LABEL: f_thunk: ; WINDOWS-NOT: mov{{.}}ps ; WINDOWS-DAG: movq %rdx, {{.*}} @@ -92,6 +127,10 @@ define void @g_thunk(i8* %fptr_i8, ...) { ; LINUX-NOT: movq ; LINUX: jmpq *%rdi # TAILCALL +; LINUX-X32-LABEL: g_thunk: +; LINUX-X32-DAG: movl %edi, %[[REG:e[abcd]x|ebp|esi|edi|r8|r9|r1[0-5]]] +; LINUX-X32-DAG: jmpq *%[[REG]] # TAILCALL + ; WINDOWS-LABEL: g_thunk: ; WINDOWS-NOT: movq ; WINDOWS: jmpq *%rcx # TAILCALL @@ -130,6 +169,10 @@ else: ; LINUX: jne ; LINUX: jmpq *{{.*}} # TAILCALL ; LINUX: jmpq *{{.*}} # TAILCALL +; LINUX-X32-LABEL: h_thunk: +; LINUX-X32: jne +; LINUX-X32: jmpq *{{.*}} # TAILCALL +; LINUX-X32: jmpq *{{.*}} # TAILCALL ; WINDOWS-LABEL: h_thunk: ; WINDOWS: jne ; WINDOWS: jmpq *{{.*}} # TAILCALL diff --git a/test/CodeGen/X86/nontemporal-2.ll b/test/CodeGen/X86/nontemporal-2.ll index 8c08b3c163c0..c9767f88488c 100644 --- a/test/CodeGen/X86/nontemporal-2.ll +++ b/test/CodeGen/X86/nontemporal-2.ll @@ -1,6 +1,6 @@ -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=corei7 | FileCheck %s -check-prefix=CHECK -check-prefix=SSE -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=corei7-avx | FileCheck %s -check-prefix=CHECK -check-prefix=AVX -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=core-avx2 | FileCheck %s -check-prefix=CHECK -check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s -check-prefix=CHECK -check-prefix=SSE +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s -check-prefix=CHECK -check-prefix=AVX +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s -check-prefix=CHECK -check-prefix=AVX2 ; Make sure that we generate non-temporal stores for the test cases below. ; We use xorps for zeroing, so domain information isn't available anymore. @@ -300,4 +300,19 @@ define void @test_op_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8>* %dst) { ret void } +; 256-bit NT stores require 256-bit alignment. +; FIXME: For AVX, we could lower this to 2x movntps %xmm. Taken further, we +; could even scalarize to movnti when we have 1-alignment: nontemporal is +; probably always worth even some 20 instruction scalarization. +define void @test_unaligned_v8f32(<8 x float> %a, <8 x float> %b, <8 x float>* %dst) { +; CHECK-LABEL: test_unaligned_v8f32: +; SSE: movntps %xmm +; SSE: movntps %xmm +; AVX-NOT: movnt +; AVX: vmovups %ymm + %r = fadd <8 x float> %a, %b + store <8 x float> %r, <8 x float>* %dst, align 16, !nontemporal !1 + ret void +} + !1 = !{i32 1} diff --git a/test/CodeGen/X86/nontemporal.ll b/test/CodeGen/X86/nontemporal.ll index f9385df36421..9a2f23596f79 100644 --- a/test/CodeGen/X86/nontemporal.ll +++ b/test/CodeGen/X86/nontemporal.ll @@ -1,6 +1,6 @@ -; RUN: llc < %s -march=x86 -mattr=+sse2 | FileCheck %s +; RUN: llc < %s -mtriple x86_64-unknown-unknown | FileCheck %s -define void @f(<4 x float> %A, i8* %B, <2 x double> %C, i32 %D, <2 x i64> %E) { +define void @f(<4 x float> %A, i8* %B, <2 x double> %C, i32 %D, <2 x i64> %E, i64 %F) { ; CHECK: movntps %cast = bitcast i8* %B to <4 x float>* %A2 = fadd <4 x float> %A, @@ -13,9 +13,12 @@ define void @f(<4 x float> %A, i8* %B, <2 x double> %C, i32 %D, <2 x i64> %E) { %cast2 = bitcast i8* %B to <2 x double>* %C2 = fadd <2 x double> %C, store <2 x double> %C2, <2 x double>* %cast2, align 16, !nontemporal !0 -; CHECK: movnti +; CHECK: movntil %cast3 = bitcast i8* %B to i32* - store i32 %D, i32* %cast3, align 16, !nontemporal !0 + store i32 %D, i32* %cast3, align 1, !nontemporal !0 +; CHECK: movntiq + %cast4 = bitcast i8* %B to i64* + store i64 %F, i64* %cast4, align 1, !nontemporal !0 ret void } diff --git a/test/CodeGen/X86/null-streamer.ll b/test/CodeGen/X86/null-streamer.ll index 3f5abfd40f29..e80f3fcbe58d 100644 --- a/test/CodeGen/X86/null-streamer.ll +++ b/test/CodeGen/X86/null-streamer.ll @@ -14,11 +14,11 @@ define void @f1() { !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!11, !13} -!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: " ", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !9, imports: !2) +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: " ", isOptimized: true, emissionKind: 0, file: !1, enums: !2, retainedTypes: !2, subprograms: !3, globals: !9, imports: !2) !1 = !DIFile(filename: "file.c", directory: "") !2 = !{} !3 = !{!4} -!4 = !DISubprogram(name: "", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !1, type: !6, function: i32 ()* null, variables: !2) +!4 = distinct !DISubprogram(name: "", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !1, type: !6, variables: !2) !6 = !DISubroutineType(types: !7) !7 = !{!8} !8 = !DIBasicType(tag: DW_TAG_base_type, size: 32, align: 32, encoding: DW_ATE_signed) diff --git a/test/CodeGen/X86/opt-ext-uses.ll b/test/CodeGen/X86/opt-ext-uses.ll index 5d05ad9c4544..39e6fd0e6a59 100644 --- a/test/CodeGen/X86/opt-ext-uses.ll +++ b/test/CodeGen/X86/opt-ext-uses.ll @@ -1,4 +1,10 @@ -; RUN: llc < %s -march=x86 | grep movw | count 1 +; RUN: llc < %s -march=x86 | FileCheck %s + +; This test should get one and only one register to register mov. +; CHECK-LABEL: t: +; CHECK: movw +; CHECK-NOT: movw +; CHECK: ret define signext i16 @t() { entry: diff --git a/test/CodeGen/X86/or-branch.ll b/test/CodeGen/X86/or-branch.ll index 9db948adb465..4899a0fc7e88 100644 --- a/test/CodeGen/X86/or-branch.ll +++ b/test/CodeGen/X86/or-branch.ll @@ -1,5 +1,5 @@ -; RUN: llc < %s -mtriple=i386-unknown-unknown -jump-is-expensive=0 | FileCheck %s --check-prefix=JUMP2 -; RUN: llc < %s -mtriple=i386-unknown-unknown -jump-is-expensive=1 | FileCheck %s --check-prefix=JUMP1 +; RUN: llc < %s -mtriple=i386-unknown-unknown -jump-is-expensive=0 | FileCheck %s --check-prefix=JUMP2 --check-prefix=CHECK +; RUN: llc < %s -mtriple=i386-unknown-unknown -jump-is-expensive=1 | FileCheck %s --check-prefix=JUMP1 --check-prefix=CHECK define void @foo(i32 %X, i32 %Y, i32 %Z) nounwind { ; JUMP2-LABEL: foo: @@ -25,4 +25,30 @@ UnifiedReturnBlock: ret void } +; If the branch is unpredictable, don't add another branch +; regardless of whether they are expensive or not. + +define void @unpredictable(i32 %X, i32 %Y, i32 %Z) nounwind { +; CHECK-LABEL: unpredictable: +; CHECK-DAG: sete +; CHECK-DAG: setl +; CHECK: orb +; CHECK: jne +entry: + %tmp1 = icmp eq i32 %X, 0 + %tmp3 = icmp slt i32 %Y, 5 + %tmp4 = or i1 %tmp3, %tmp1 + br i1 %tmp4, label %cond_true, label %UnifiedReturnBlock, !unpredictable !0 + +cond_true: + %tmp5 = tail call i32 (...) @bar( ) + ret void + +UnifiedReturnBlock: + ret void +} + declare i32 @bar(...) + +!0 = !{} + diff --git a/test/CodeGen/X86/or-lea.ll b/test/CodeGen/X86/or-lea.ll new file mode 100644 index 000000000000..f45a639ffa2c --- /dev/null +++ b/test/CodeGen/X86/or-lea.ll @@ -0,0 +1,120 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s + +; InstCombine and DAGCombiner transform an 'add' into an 'or' +; if there are no common bits from the incoming operands. +; LEA instruction selection should be able to see through that +; transform and reduce add/shift/or instruction counts. + +define i32 @or_shift1_and1(i32 %x, i32 %y) { +; CHECK-LABEL: or_shift1_and1: +; CHECK: # BB#0: +; CHECK-NEXT: andl $1, %esi +; CHECK-NEXT: leal (%rsi,%rdi,2), %eax +; CHECK-NEXT: retq + + %shl = shl i32 %x, 1 + %and = and i32 %y, 1 + %or = or i32 %and, %shl + ret i32 %or +} + +define i32 @or_shift1_and1_swapped(i32 %x, i32 %y) { +; CHECK-LABEL: or_shift1_and1_swapped: +; CHECK: # BB#0: +; CHECK-NEXT: andl $1, %esi +; CHECK-NEXT: leal (%rsi,%rdi,2), %eax +; CHECK-NEXT: retq + + %shl = shl i32 %x, 1 + %and = and i32 %y, 1 + %or = or i32 %shl, %and + ret i32 %or +} + +define i32 @or_shift2_and1(i32 %x, i32 %y) { +; CHECK-LABEL: or_shift2_and1: +; CHECK: # BB#0: +; CHECK-NEXT: andl $1, %esi +; CHECK-NEXT: leal (%rsi,%rdi,4), %eax +; CHECK-NEXT: retq + + %shl = shl i32 %x, 2 + %and = and i32 %y, 1 + %or = or i32 %shl, %and + ret i32 %or +} + +define i32 @or_shift3_and1(i32 %x, i32 %y) { +; CHECK-LABEL: or_shift3_and1: +; CHECK: # BB#0: +; CHECK-NEXT: andl $1, %esi +; CHECK-NEXT: leal (%rsi,%rdi,8), %eax +; CHECK-NEXT: retq + + %shl = shl i32 %x, 3 + %and = and i32 %y, 1 + %or = or i32 %shl, %and + ret i32 %or +} + +define i32 @or_shift3_and7(i32 %x, i32 %y) { +; CHECK-LABEL: or_shift3_and7: +; CHECK: # BB#0: +; CHECK-NEXT: andl $7, %esi +; CHECK-NEXT: leal (%rsi,%rdi,8), %eax +; CHECK-NEXT: retq + + %shl = shl i32 %x, 3 + %and = and i32 %y, 7 + %or = or i32 %shl, %and + ret i32 %or +} + +; The shift is too big for an LEA. + +define i32 @or_shift4_and1(i32 %x, i32 %y) { +; CHECK-LABEL: or_shift4_and1: +; CHECK: # BB#0: +; CHECK-NEXT: shll $4, %edi +; CHECK-NEXT: andl $1, %esi +; CHECK-NEXT: leal (%rsi,%rdi), %eax +; CHECK-NEXT: retq + + %shl = shl i32 %x, 4 + %and = and i32 %y, 1 + %or = or i32 %shl, %and + ret i32 %or +} + +; The mask is too big for the shift, so the 'or' isn't equivalent to an 'add'. + +define i32 @or_shift3_and8(i32 %x, i32 %y) { +; CHECK-LABEL: or_shift3_and8: +; CHECK: # BB#0: +; CHECK-NEXT: leal (,%rdi,8), %eax +; CHECK-NEXT: andl $8, %esi +; CHECK-NEXT: orl %esi, %eax +; CHECK-NEXT: retq + + %shl = shl i32 %x, 3 + %and = and i32 %y, 8 + %or = or i32 %shl, %and + ret i32 %or +} + +; 64-bit operands should work too. + +define i64 @or_shift1_and1_64(i64 %x, i64 %y) { +; CHECK-LABEL: or_shift1_and1_64: +; CHECK: # BB#0: +; CHECK-NEXT: andl $1, %esi +; CHECK-NEXT: leaq (%rsi,%rdi,2), %rax +; CHECK-NEXT: retq + + %shl = shl i64 %x, 1 + %and = and i64 %y, 1 + %or = or i64 %and, %shl + ret i64 %or +} + diff --git a/test/CodeGen/X86/palignr.ll b/test/CodeGen/X86/palignr.ll index dfa2cedf45a2..d75506cadfa2 100644 --- a/test/CodeGen/X86/palignr.ll +++ b/test/CodeGen/X86/palignr.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -march=x86 -mcpu=core2 -mattr=+ssse3 | FileCheck %s ; RUN: llc < %s -march=x86 -mcpu=yonah | FileCheck --check-prefix=CHECK-YONAH %s diff --git a/test/CodeGen/X86/patchpoint-verifiable.mir b/test/CodeGen/X86/patchpoint-verifiable.mir new file mode 100644 index 000000000000..300ecaf002f2 --- /dev/null +++ b/test/CodeGen/X86/patchpoint-verifiable.mir @@ -0,0 +1,42 @@ +# RUN: llc -mtriple=x86_64-apple-darwin -stop-after branch-folder -start-after branch-folder -o /dev/null %s 2>&1 | FileCheck %s +# This test verifies that the machine verifier won't report an error when +# verifying the PATCHPOINT instruction. + +--- | + + define void @small_patchpoint_codegen(i64 %p1, i64 %p2, i64 %p3, i64 %p4) { + entry: + %result = tail call i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 5, i32 5, i8* null, i32 2, i64 %p1, i64 %p2) + ret void + } + + declare i64 @llvm.experimental.patchpoint.i64(i64, i32, i8*, i32, ...) + +... +--- +name: small_patchpoint_codegen +tracksRegLiveness: true +liveins: + - { reg: '%rdi' } + - { reg: '%rsi' } +frameInfo: + hasPatchPoint: true + stackSize: 8 + adjustsStack: true + hasCalls: true +fixedStack: + - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 16 } +body: | + bb.0.entry: + liveins: %rdi, %rsi, %rbp + + frame-setup PUSH64r killed %rbp, implicit-def %rsp, implicit %rsp + CFI_INSTRUCTION .cfi_def_cfa_offset 16 + CFI_INSTRUCTION .cfi_offset %rbp, -16 + %rbp = frame-setup MOV64rr %rsp + CFI_INSTRUCTION .cfi_def_cfa_register %rbp + ; CHECK: PATCHPOINT 5, 5, 0, 2, 0, %rdi, %rsi, csr_64, implicit-def dead early-clobber %r11, implicit-def %rsp, implicit-def dead %rax + PATCHPOINT 5, 5, 0, 2, 0, %rdi, %rsi, csr_64, implicit-def dead early-clobber %r11, implicit-def %rsp, implicit-def dead %rax + %rbp = POP64r implicit-def %rsp, implicit %rsp + RETQ +... diff --git a/test/CodeGen/X86/peephole-na-phys-copy-folding.ll b/test/CodeGen/X86/peephole-na-phys-copy-folding.ll new file mode 100644 index 000000000000..bf457814079c --- /dev/null +++ b/test/CodeGen/X86/peephole-na-phys-copy-folding.ll @@ -0,0 +1,190 @@ +; RUN: llc -verify-machineinstrs -mtriple=i386-linux-gnu %s -o - | FileCheck %s +; RUN: llc -verify-machineinstrs -mtriple=x86_64-linux-gnu -mattr=+sahf %s -o - | FileCheck %s + +; The peephole optimizer can elide some physical register copies such as +; EFLAGS. Make sure the flags are used directly, instead of needlessly using +; lahf, when possible. + +@L = external global i32 +@M = external global i8 +declare i32 @bar(i64) + +; CHECK-LABEL: plus_one +; CHECK-NOT: seto +; CHECK-NOT: lahf +; CHECK-NOT: sahf +; CHECK-NOT: pushf +; CHECK-NOT: popf +; CHECK: incl L +define i1 @plus_one() { +entry: + %loaded_L = load i32, i32* @L + %val = add nsw i32 %loaded_L, 1 ; N.B. will emit inc. + store i32 %val, i32* @L + %loaded_M = load i8, i8* @M + %masked = and i8 %loaded_M, 8 + %M_is_true = icmp ne i8 %masked, 0 + %L_is_false = icmp eq i32 %val, 0 + %cond = and i1 %L_is_false, %M_is_true + br i1 %cond, label %exit2, label %exit + +exit: + ret i1 true + +exit2: + ret i1 false +} + +; CHECK-LABEL: plus_forty_two +; CHECK-NOT: seto +; CHECK-NOT: lahf +; CHECK-NOT: sahf +; CHECK-NOT: pushf +; CHECK-NOT: popf +; CHECK: addl $42, +define i1 @plus_forty_two() { +entry: + %loaded_L = load i32, i32* @L + %val = add nsw i32 %loaded_L, 42 ; N.B. won't emit inc. + store i32 %val, i32* @L + %loaded_M = load i8, i8* @M + %masked = and i8 %loaded_M, 8 + %M_is_true = icmp ne i8 %masked, 0 + %L_is_false = icmp eq i32 %val, 0 + %cond = and i1 %L_is_false, %M_is_true + br i1 %cond, label %exit2, label %exit + +exit: + ret i1 true + +exit2: + ret i1 false +} + +; CHECK-LABEL: minus_one +; CHECK-NOT: seto +; CHECK-NOT: lahf +; CHECK-NOT: sahf +; CHECK-NOT: pushf +; CHECK-NOT: popf +; CHECK: decl L +define i1 @minus_one() { +entry: + %loaded_L = load i32, i32* @L + %val = add nsw i32 %loaded_L, -1 ; N.B. will emit dec. + store i32 %val, i32* @L + %loaded_M = load i8, i8* @M + %masked = and i8 %loaded_M, 8 + %M_is_true = icmp ne i8 %masked, 0 + %L_is_false = icmp eq i32 %val, 0 + %cond = and i1 %L_is_false, %M_is_true + br i1 %cond, label %exit2, label %exit + +exit: + ret i1 true + +exit2: + ret i1 false +} + +; CHECK-LABEL: minus_forty_two +; CHECK-NOT: seto +; CHECK-NOT: lahf +; CHECK-NOT: sahf +; CHECK-NOT: pushf +; CHECK-NOT: popf +; CHECK: addl $-42, +define i1 @minus_forty_two() { +entry: + %loaded_L = load i32, i32* @L + %val = add nsw i32 %loaded_L, -42 ; N.B. won't emit dec. + store i32 %val, i32* @L + %loaded_M = load i8, i8* @M + %masked = and i8 %loaded_M, 8 + %M_is_true = icmp ne i8 %masked, 0 + %L_is_false = icmp eq i32 %val, 0 + %cond = and i1 %L_is_false, %M_is_true + br i1 %cond, label %exit2, label %exit + +exit: + ret i1 true + +exit2: + ret i1 false +} + +; CHECK-LABEL: test_intervening_call: +; CHECK: cmpxchg +; CHECK: seto %al +; CHECK-NEXT: lahf +; CHECK: call{{[lq]}} bar +; CHECK: addb $127, %al +; CHECK-NEXT: sahf +define i64 @test_intervening_call(i64* %foo, i64 %bar, i64 %baz) { + ; cmpxchg sets EFLAGS, call clobbers it, then br uses EFLAGS. + %cx = cmpxchg i64* %foo, i64 %bar, i64 %baz seq_cst seq_cst + %v = extractvalue { i64, i1 } %cx, 0 + %p = extractvalue { i64, i1 } %cx, 1 + call i32 @bar(i64 %v) + br i1 %p, label %t, label %f + +t: + ret i64 42 + +f: + ret i64 0 +} + +; CHECK-LABEL: test_two_live_flags: +; CHECK: cmpxchg +; CHECK: seto %al +; CHECK-NEXT: lahf +; Save result of the first cmpxchg into D. +; CHECK-NEXT: mov{{[lq]}} %[[AX:[er]ax]], %[[D:[re]d[xi]]] +; CHECK: cmpxchg +; CHECK-NEXT: sete %al +; Save result of the second cmpxchg onto the stack. +; CHECK-NEXT: push{{[lq]}} %[[AX]] +; Restore result of the first cmpxchg from D, put it back in EFLAGS. +; CHECK-NEXT: mov{{[lq]}} %[[D]], %[[AX]] +; CHECK-NEXT: addb $127, %al +; CHECK-NEXT: sahf +; Restore result of the second cmpxchg from the stack. +; CHECK-NEXT: pop{{[lq]}} %[[AX]] +; Test from EFLAGS restored from first cmpxchg, jump if that fails. +; CHECK-NEXT: jne +; Fallthrough to test the second cmpxchg's result. +; CHECK: testb %al, %al +; CHECK-NEXT: je +define i64 @test_two_live_flags( + i64* %foo0, i64 %bar0, i64 %baz0, + i64* %foo1, i64 %bar1, i64 %baz1) { + %cx0 = cmpxchg i64* %foo0, i64 %bar0, i64 %baz0 seq_cst seq_cst + %p0 = extractvalue { i64, i1 } %cx0, 1 + %cx1 = cmpxchg i64* %foo1, i64 %bar1, i64 %baz1 seq_cst seq_cst + %p1 = extractvalue { i64, i1 } %cx1, 1 + %flag = and i1 %p0, %p1 + br i1 %flag, label %t, label %f + +t: + ret i64 42 + +f: + ret i64 0 +} + +; CHECK-LABEL: asm_clobbering_flags: +; CHECK: test +; CHECK-NEXT: setg +; CHECK-NEXT: #APP +; CHECK-NEXT: bsfl +; CHECK-NEXT: #NO_APP +; CHECK-NEXT: movl +; CHECK-NEXT: ret +define i1 @asm_clobbering_flags(i32* %mem) { + %val = load i32, i32* %mem, align 4 + %cmp = icmp sgt i32 %val, 0 + %res = tail call i32 asm "bsfl $1,$0", "=r,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i32 %val) + store i32 %res, i32* %mem, align 4 + ret i1 %cmp +} diff --git a/test/CodeGen/X86/pmul.ll b/test/CodeGen/X86/pmul.ll index dbe5bd646c7f..37b6fdf7cfeb 100644 --- a/test/CodeGen/X86/pmul.ll +++ b/test/CodeGen/X86/pmul.ll @@ -1,6 +1,7 @@ -; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=ALL --check-prefix=SSE2 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE41 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=core-avx2 | FileCheck %s --check-prefix=AVX2 +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX2 define <16 x i8> @mul8c(<16 x i8> %i) nounwind { ; SSE2-LABEL: mul8c: @@ -34,16 +35,34 @@ define <16 x i8> @mul8c(<16 x i8> %i) nounwind { ; SSE41-NEXT: packuswb %xmm0, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: retq +; +; AVX2-LABEL: mul8c: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovsxbw %xmm0, %ymm0 +; AVX2-NEXT: vpmovsxbw {{.*}}(%rip), %ymm1 +; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u> +; AVX2-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; AVX2-NEXT: vpshufb %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq entry: %A = mul <16 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 > ret <16 x i8> %A } define <8 x i16> @mul16c(<8 x i16> %i) nounwind { -; ALL-LABEL: mul16c: -; ALL: # BB#0: # %entry -; ALL-NEXT: pmullw {{.*}}(%rip), %xmm0 -; ALL-NEXT: retq +; SSE-LABEL: mul16c: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmullw {{.*}}(%rip), %xmm0 +; SSE-NEXT: retq +; +; AVX2-LABEL: mul16c: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmullw {{.*}}(%rip), %xmm0, %xmm0 +; AVX2-NEXT: retq entry: %A = mul <8 x i16> %i, < i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117 > ret <8 x i16> %A @@ -65,22 +84,38 @@ define <4 x i32> @a(<4 x i32> %i) nounwind { ; SSE41: # BB#0: # %entry ; SSE41-NEXT: pmulld {{.*}}(%rip), %xmm0 ; SSE41-NEXT: retq +; +; AVX2-LABEL: a: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %xmm1 +; AVX2-NEXT: vpmulld %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq entry: %A = mul <4 x i32> %i, < i32 117, i32 117, i32 117, i32 117 > ret <4 x i32> %A } define <2 x i64> @b(<2 x i64> %i) nounwind { -; ALL-LABEL: b: -; ALL: # BB#0: # %entry -; ALL-NEXT: movdqa {{.*#+}} xmm1 = [117,117] -; ALL-NEXT: movdqa %xmm0, %xmm2 -; ALL-NEXT: pmuludq %xmm1, %xmm2 -; ALL-NEXT: psrlq $32, %xmm0 -; ALL-NEXT: pmuludq %xmm1, %xmm0 -; ALL-NEXT: psllq $32, %xmm0 -; ALL-NEXT: paddq %xmm2, %xmm0 -; ALL-NEXT: retq +; SSE-LABEL: b: +; SSE: # BB#0: # %entry +; SSE-NEXT: movdqa {{.*#+}} xmm1 = [117,117] +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: pmuludq %xmm1, %xmm2 +; SSE-NEXT: psrlq $32, %xmm0 +; SSE-NEXT: pmuludq %xmm1, %xmm0 +; SSE-NEXT: psllq $32, %xmm0 +; SSE-NEXT: paddq %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX2-LABEL: b: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [117,117] +; AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm2 +; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm0 +; AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpsllq $32, %xmm0, %xmm0 +; AVX2-NEXT: vpaddq %xmm0, %xmm2, %xmm0 +; AVX2-NEXT: retq entry: %A = mul <2 x i64> %i, < i64 117, i64 117 > ret <2 x i64> %A @@ -123,16 +158,34 @@ define <16 x i8> @mul8(<16 x i8> %i, <16 x i8> %j) nounwind { ; SSE41-NEXT: packuswb %xmm0, %xmm2 ; SSE41-NEXT: movdqa %xmm2, %xmm0 ; SSE41-NEXT: retq +; +; AVX2-LABEL: mul8: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovsxbw %xmm1, %ymm1 +; AVX2-NEXT: vpmovsxbw %xmm0, %ymm0 +; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u> +; AVX2-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; AVX2-NEXT: vpshufb %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq entry: %A = mul <16 x i8> %i, %j ret <16 x i8> %A } define <8 x i16> @mul16(<8 x i16> %i, <8 x i16> %j) nounwind { -; ALL-LABEL: mul16: -; ALL: # BB#0: # %entry -; ALL-NEXT: pmullw %xmm1, %xmm0 -; ALL-NEXT: retq +; SSE-LABEL: mul16: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmullw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX2-LABEL: mul16: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq entry: %A = mul <8 x i16> %i, %j ret <8 x i16> %A @@ -154,26 +207,44 @@ define <4 x i32> @c(<4 x i32> %i, <4 x i32> %j) nounwind { ; SSE41: # BB#0: # %entry ; SSE41-NEXT: pmulld %xmm1, %xmm0 ; SSE41-NEXT: retq +; +; AVX2-LABEL: c: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmulld %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq entry: %A = mul <4 x i32> %i, %j ret <4 x i32> %A } define <2 x i64> @d(<2 x i64> %i, <2 x i64> %j) nounwind { -; ALL-LABEL: d: -; ALL: # BB#0: # %entry -; ALL-NEXT: movdqa %xmm0, %xmm2 -; ALL-NEXT: pmuludq %xmm1, %xmm2 -; ALL-NEXT: movdqa %xmm1, %xmm3 -; ALL-NEXT: psrlq $32, %xmm3 -; ALL-NEXT: pmuludq %xmm0, %xmm3 -; ALL-NEXT: psllq $32, %xmm3 -; ALL-NEXT: paddq %xmm3, %xmm2 -; ALL-NEXT: psrlq $32, %xmm0 -; ALL-NEXT: pmuludq %xmm1, %xmm0 -; ALL-NEXT: psllq $32, %xmm0 -; ALL-NEXT: paddq %xmm2, %xmm0 -; ALL-NEXT: retq +; SSE-LABEL: d: +; SSE: # BB#0: # %entry +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: pmuludq %xmm1, %xmm2 +; SSE-NEXT: movdqa %xmm1, %xmm3 +; SSE-NEXT: psrlq $32, %xmm3 +; SSE-NEXT: pmuludq %xmm0, %xmm3 +; SSE-NEXT: psllq $32, %xmm3 +; SSE-NEXT: paddq %xmm3, %xmm2 +; SSE-NEXT: psrlq $32, %xmm0 +; SSE-NEXT: pmuludq %xmm1, %xmm0 +; SSE-NEXT: psllq $32, %xmm0 +; SSE-NEXT: paddq %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX2-LABEL: d: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm2 +; AVX2-NEXT: vpsrlq $32, %xmm1, %xmm3 +; AVX2-NEXT: vpmuludq %xmm3, %xmm0, %xmm3 +; AVX2-NEXT: vpsllq $32, %xmm3, %xmm3 +; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vpsrlq $32, %xmm0, %xmm0 +; AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpsllq $32, %xmm0, %xmm0 +; AVX2-NEXT: vpaddq %xmm0, %xmm2, %xmm0 +; AVX2-NEXT: retq entry: %A = mul <2 x i64> %i, %j ret <2 x i64> %A @@ -210,6 +281,17 @@ define <4 x i32> @e(<4 x i32> %i, <4 x i32> %j) nounwind { ; SSE41-NEXT: pmulld {{[0-9]+}}(%rsp), %xmm0 # 16-byte Folded Reload ; SSE41-NEXT: addq $40, %rsp ; SSE41-NEXT: retq +; +; AVX2-LABEL: e: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: subq $40, %rsp +; AVX2-NEXT: vmovaps %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill +; AVX2-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; AVX2-NEXT: callq foo +; AVX2-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload +; AVX2-NEXT: vpmulld {{[0-9]+}}(%rsp), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX2-NEXT: addq $40, %rsp +; AVX2-NEXT: retq entry: ; Use a call to force spills. call void @foo() @@ -218,27 +300,47 @@ entry: } define <2 x i64> @f(<2 x i64> %i, <2 x i64> %j) nounwind { -; ALL-LABEL: f: -; ALL: # BB#0: # %entry -; ALL-NEXT: subq $40, %rsp -; ALL-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill -; ALL-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill -; ALL-NEXT: callq foo -; ALL-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload -; ALL-NEXT: movdqa %xmm0, %xmm2 -; ALL-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm3 # 16-byte Reload -; ALL-NEXT: pmuludq %xmm3, %xmm2 -; ALL-NEXT: movdqa %xmm3, %xmm1 -; ALL-NEXT: psrlq $32, %xmm1 -; ALL-NEXT: pmuludq %xmm0, %xmm1 -; ALL-NEXT: psllq $32, %xmm1 -; ALL-NEXT: paddq %xmm1, %xmm2 -; ALL-NEXT: psrlq $32, %xmm0 -; ALL-NEXT: pmuludq %xmm3, %xmm0 -; ALL-NEXT: psllq $32, %xmm0 -; ALL-NEXT: paddq %xmm2, %xmm0 -; ALL-NEXT: addq $40, %rsp -; ALL-NEXT: retq +; SSE-LABEL: f: +; SSE: # BB#0: # %entry +; SSE-NEXT: subq $40, %rsp +; SSE-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill +; SSE-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill +; SSE-NEXT: callq foo +; SSE-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm3 # 16-byte Reload +; SSE-NEXT: pmuludq %xmm3, %xmm2 +; SSE-NEXT: movdqa %xmm3, %xmm1 +; SSE-NEXT: psrlq $32, %xmm1 +; SSE-NEXT: pmuludq %xmm0, %xmm1 +; SSE-NEXT: psllq $32, %xmm1 +; SSE-NEXT: paddq %xmm1, %xmm2 +; SSE-NEXT: psrlq $32, %xmm0 +; SSE-NEXT: pmuludq %xmm3, %xmm0 +; SSE-NEXT: psllq $32, %xmm0 +; SSE-NEXT: paddq %xmm2, %xmm0 +; SSE-NEXT: addq $40, %rsp +; SSE-NEXT: retq +; +; AVX2-LABEL: f: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: subq $40, %rsp +; AVX2-NEXT: vmovaps %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill +; AVX2-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; AVX2-NEXT: callq foo +; AVX2-NEXT: vmovdqa {{[0-9]+}}(%rsp), %xmm2 # 16-byte Reload +; AVX2-NEXT: vmovdqa (%rsp), %xmm3 # 16-byte Reload +; AVX2-NEXT: vpmuludq %xmm2, %xmm3, %xmm0 +; AVX2-NEXT: vpsrlq $32, %xmm2, %xmm1 +; AVX2-NEXT: vpmuludq %xmm1, %xmm3, %xmm1 +; AVX2-NEXT: vpsllq $32, %xmm1, %xmm1 +; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpsrlq $32, %xmm3, %xmm1 +; AVX2-NEXT: vpmuludq %xmm2, %xmm1, %xmm1 +; AVX2-NEXT: vpsllq $32, %xmm1, %xmm1 +; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: addq $40, %rsp +; AVX2-NEXT: retq entry: ; Use a call to force spills. call void @foo() @@ -247,31 +349,76 @@ entry: } define <4 x i64> @b1(<4 x i64> %i) nounwind { -; AVX2-LABEL: @b1 -; AVX2: vpbroadcastq -; AVX2-NEXT: vpmuludq -; AVX2-NEXT: vpsrlq $32 -; AVX2-NEXT: vpmuludq -; AVX2-NEXT: vpsllq $32 -; AVX2-NEXT: vpaddq -; AVX2-NEXT: retq +; SSE-LABEL: b1: +; SSE: # BB#0: # %entry +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [117,117] +; SSE-NEXT: movdqa %xmm0, %xmm3 +; SSE-NEXT: pmuludq %xmm2, %xmm3 +; SSE-NEXT: psrlq $32, %xmm0 +; SSE-NEXT: pmuludq %xmm2, %xmm0 +; SSE-NEXT: psllq $32, %xmm0 +; SSE-NEXT: paddq %xmm3, %xmm0 +; SSE-NEXT: movdqa %xmm1, %xmm3 +; SSE-NEXT: pmuludq %xmm2, %xmm3 +; SSE-NEXT: psrlq $32, %xmm1 +; SSE-NEXT: pmuludq %xmm2, %xmm1 +; SSE-NEXT: psllq $32, %xmm1 +; SSE-NEXT: paddq %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX2-LABEL: b1: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm1 +; AVX2-NEXT: vpmuludq %ymm1, %ymm0, %ymm2 +; AVX2-NEXT: vpsrlq $32, %ymm0, %ymm0 +; AVX2-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpsllq $32, %ymm0, %ymm0 +; AVX2-NEXT: vpaddq %ymm0, %ymm2, %ymm0 +; AVX2-NEXT: retq entry: %A = mul <4 x i64> %i, < i64 117, i64 117, i64 117, i64 117 > ret <4 x i64> %A } define <4 x i64> @b2(<4 x i64> %i, <4 x i64> %j) nounwind { -; AVX2-LABEL: @b2 -; AVX2: vpmuludq -; AVX2-NEXT: vpsrlq $32 -; AVX2-NEXT: vpmuludq -; AVX2-NEXT: vpsllq $32 -; AVX2-NEXT: vpaddq -; AVX2-NEXT: vpsrlq $32 -; AVX2-NEXT: vpmuludq -; AVX2-NEXT: vpsllq $32 -; AVX2-NEXT: vpaddq -; AVX2-NEXT: retq +; SSE-LABEL: b2: +; SSE: # BB#0: # %entry +; SSE-NEXT: movdqa %xmm0, %xmm4 +; SSE-NEXT: pmuludq %xmm2, %xmm4 +; SSE-NEXT: movdqa %xmm2, %xmm5 +; SSE-NEXT: psrlq $32, %xmm5 +; SSE-NEXT: pmuludq %xmm0, %xmm5 +; SSE-NEXT: psllq $32, %xmm5 +; SSE-NEXT: paddq %xmm5, %xmm4 +; SSE-NEXT: psrlq $32, %xmm0 +; SSE-NEXT: pmuludq %xmm2, %xmm0 +; SSE-NEXT: psllq $32, %xmm0 +; SSE-NEXT: paddq %xmm4, %xmm0 +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: pmuludq %xmm3, %xmm2 +; SSE-NEXT: movdqa %xmm3, %xmm4 +; SSE-NEXT: psrlq $32, %xmm4 +; SSE-NEXT: pmuludq %xmm1, %xmm4 +; SSE-NEXT: psllq $32, %xmm4 +; SSE-NEXT: paddq %xmm4, %xmm2 +; SSE-NEXT: psrlq $32, %xmm1 +; SSE-NEXT: pmuludq %xmm3, %xmm1 +; SSE-NEXT: psllq $32, %xmm1 +; SSE-NEXT: paddq %xmm2, %xmm1 +; SSE-NEXT: retq +; +; AVX2-LABEL: b2: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmuludq %ymm1, %ymm0, %ymm2 +; AVX2-NEXT: vpsrlq $32, %ymm1, %ymm3 +; AVX2-NEXT: vpmuludq %ymm3, %ymm0, %ymm3 +; AVX2-NEXT: vpsllq $32, %ymm3, %ymm3 +; AVX2-NEXT: vpaddq %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vpsrlq $32, %ymm0, %ymm0 +; AVX2-NEXT: vpmuludq %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpsllq $32, %ymm0, %ymm0 +; AVX2-NEXT: vpaddq %ymm0, %ymm2, %ymm0 +; AVX2-NEXT: retq entry: %A = mul <4 x i64> %i, %j ret <4 x i64> %A diff --git a/test/CodeGen/X86/pop-stack-cleanup.ll b/test/CodeGen/X86/pop-stack-cleanup.ll new file mode 100644 index 000000000000..bcf7594065f3 --- /dev/null +++ b/test/CodeGen/X86/pop-stack-cleanup.ll @@ -0,0 +1,76 @@ +; RUN: llc < %s -mtriple=i686-windows | FileCheck %s -check-prefix=CHECK +; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s -check-prefix=LINUX64 + +declare void @param1(i32 %a) +declare i32 @param2_ret(i32 %a, i32 %b) +declare i64 @param2_ret64(i32 %a, i32 %b) +declare void @param2(i32 %a, i32 %b) +declare void @param3(i32 %a, i32 %b, i32 %c) +declare void @param8(i64, i64, i64, i64, i64, i64, i64, i64) + + +define void @test() minsize nounwind { +; CHECK-LABEL: test: +; CHECK: calll _param1 +; CHECK-NEXT: popl %eax +; CHECK: calll _param2 +; CHECK-NEXT: popl %eax +; CHECK-NEXT: popl %ecx +; CHECK: calll _param2_ret +; CHECK-NEXT: popl %ecx +; CHECK-NEXT: popl %edx +; CHECK-NEXT: pushl %eax +; CHECK: calll _param3 +; CHECK-NEXT: addl $12, %esp +; CHECK: calll _param2_ret64 +; CHECK-NEXT: popl %ecx +; CHECK-NEXT: popl %ecx + call void @param1(i32 1) + call void @param2(i32 1, i32 2) + %ret = call i32 @param2_ret(i32 1, i32 2) + call void @param3(i32 1, i32 2, i32 %ret) + %ret64 = call i64 @param2_ret64(i32 1, i32 2) + ret void +} + +define void @negative(i32 %k) { +; CHECK-LABEL: negative: +; CHECK: calll _param1 +; CHECK-NEXT: addl $4, %esp +; CHECK: calll _param2 +; CHECK-NEXT: addl $8, %esp +; CHECK: calll _param3 +; CHECK-NEXT: movl %ebp, %esp + %v = alloca i32, i32 %k + call void @param1(i32 1) + call void @param2(i32 1, i32 2) + call void @param3(i32 1, i32 2, i32 3) + ret void +} + +define void @spill(i32 inreg %a, i32 inreg %b, i32 inreg %c) minsize nounwind { +; CHECK-LABEL: spill: +; CHECK-DAG: movl %ecx, +; CHECK-DAG: movl %edx, +; CHECK: calll _param2_ret +; CHECK-NEXT: popl %ecx +; CHECK-NEXT: popl %edx +; CHECK-DAG: movl {{.*}}, %ecx +; CHECK-DAG: movl {{.*}}, %edx +; CHECK: calll _spill + %i = call i32 @param2_ret(i32 1, i32 2) + call void @spill(i32 %a, i32 %b, i32 %c) + ret void +} + +define void @test_linux64(i32 %size) minsize nounwind { +; LINUX64-LABEL: test_linux64: +; LINUX64: pushq %rbp +; LINUX64: callq param8 +; LINUX64-NEXT: popq %rax +; LINUX64-NEXT: popq %rcx + + %a = alloca i64, i32 %size, align 8 + call void @param8(i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8) + ret void +} diff --git a/test/CodeGen/X86/powi.ll b/test/CodeGen/X86/powi.ll index c3d68312ce15..17d3e3e7d33c 100644 --- a/test/CodeGen/X86/powi.ll +++ b/test/CodeGen/X86/powi.ll @@ -1,10 +1,38 @@ -; RUN: llc %s -march=x86 -mcpu=yonah -o - | grep mulsd | count 6 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse2 | FileCheck %s ; Ideally this would compile to 5 multiplies. -define double @_Z3f10d(double %a) nounwind readonly ssp noredzone { -entry: - %0 = tail call double @llvm.powi.f64(double %a, i32 15) nounwind ; [#uses=1] - ret double %0 +define double @pow_wrapper(double %a) nounwind readonly ssp noredzone { +; CHECK-LABEL: pow_wrapper: +; CHECK: # BB#0: +; CHECK-NEXT: movapd %xmm0, %xmm1 +; CHECK-NEXT: mulsd %xmm1, %xmm1 +; CHECK-NEXT: mulsd %xmm1, %xmm0 +; CHECK-NEXT: mulsd %xmm1, %xmm1 +; CHECK-NEXT: mulsd %xmm1, %xmm0 +; CHECK-NEXT: mulsd %xmm1, %xmm1 +; CHECK-NEXT: mulsd %xmm0, %xmm1 +; CHECK-NEXT: movapd %xmm1, %xmm0 +; CHECK-NEXT: retq + %ret = tail call double @llvm.powi.f64(double %a, i32 15) nounwind ; [#uses=1] + ret double %ret +} + +define double @pow_wrapper_optsize(double %a) optsize { +; CHECK-LABEL: pow_wrapper_optsize: +; CHECK: # BB#0: +; CHECK-NEXT: movl $15, %edi +; CHECK-NEXT: jmp + %ret = tail call double @llvm.powi.f64(double %a, i32 15) nounwind ; [#uses=1] + ret double %ret +} + +define double @pow_wrapper_minsize(double %a) minsize { +; CHECK-LABEL: pow_wrapper_minsize: +; CHECK: # BB#0: +; CHECK-NEXT: movl $128, %edi +; CHECK-NEXT: jmp + %ret = tail call double @llvm.powi.f64(double %a, i32 128) nounwind ; [#uses=1] + ret double %ret } declare double @llvm.powi.f64(double, i32) nounwind readonly diff --git a/test/CodeGen/X86/pr11415.ll b/test/CodeGen/X86/pr11415.ll index 6c32a2206a7e..73c497014116 100644 --- a/test/CodeGen/X86/pr11415.ll +++ b/test/CodeGen/X86/pr11415.ll @@ -4,15 +4,17 @@ ; defining %0 before it was read. This caused us to omit the ; movq -8(%rsp), %rdx +; CHECK: pushq %rax ; CHECK: #APP ; CHECK-NEXT: #NO_APP ; CHECK-NEXT: movq %rcx, %rax -; CHECK-NEXT: movq %rax, -8(%rsp) -; CHECK-NEXT: movq -8(%rsp), %rdx +; CHECK-NEXT: movq %rax, (%rsp) +; CHECK-NEXT: movq (%rsp), %rdx ; CHECK-NEXT: #APP ; CHECK-NEXT: #NO_APP ; CHECK-NEXT: movq %rdx, %rax -; CHECK-NEXT: movq %rdx, -8(%rsp) +; CHECK-NEXT: movq %rdx, (%rsp) +; CHECK-NEXT: popq %rcx ; CHECK-NEXT: ret define i64 @foo() { diff --git a/test/CodeGen/X86/pr11468.ll b/test/CodeGen/X86/pr11468.ll index f721df11586b..7a2cc5b1a60d 100644 --- a/test/CodeGen/X86/pr11468.ll +++ b/test/CodeGen/X86/pr11468.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -force-align-stack -stack-alignment=32 -march=x86-64 -mattr=+avx -mtriple=i686-apple-darwin10 | FileCheck %s +; RUN: llc < %s -stackrealign -stack-alignment=32 -march=x86-64 -mattr=+avx -mtriple=i686-apple-darwin10 | FileCheck %s ; PR11468 define void @f(i64 %sz) uwtable { diff --git a/test/CodeGen/X86/pr11985.ll b/test/CodeGen/X86/pr11985.ll index fa378502f724..aae00de112d3 100644 --- a/test/CodeGen/X86/pr11985.ll +++ b/test/CodeGen/X86/pr11985.ll @@ -1,6 +1,28 @@ -; RUN: llc < %s -mtriple=x86_64-pc-linux -mcpu=prescott | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-pc-linux -mcpu=prescott | FileCheck %s --check-prefix=PRESCOTT +; RUN: llc < %s -mtriple=x86_64-pc-linux -mcpu=nehalem | FileCheck %s --check-prefix=NEHALEM + +;;; TODO: (1) Some of the loads and stores are certainly unaligned and (2) the first load and first +;;; store overlap with the second load and second store respectively. +;;; +;;; Is either of these sequences ideal? define float @foo(i8* nocapture %buf, float %a, float %b) nounwind uwtable { +; PRESCOTT-LABEL: foo: +; PRESCOTT: # BB#0: # %entry +; PRESCOTT-NEXT: movq .Ltmp0+14(%rip), %rax +; PRESCOTT-NEXT: movq %rax, 14(%rdi) +; PRESCOTT-NEXT: movq .Ltmp0+8(%rip), %rax +; PRESCOTT-NEXT: movq %rax, 8(%rdi) +; PRESCOTT-NEXT: movq .Ltmp0(%rip), %rax +; PRESCOTT-NEXT: movq %rax, (%rdi) +; +; NEHALEM-LABEL: foo: +; NEHALEM: # BB#0: # %entry +; NEHALEM-NEXT: movq .Ltmp0+14(%rip), %rax +; NEHALEM-NEXT: movq %rax, 14(%rdi) +; NEHALEM-NEXT: movups .Ltmp0(%rip), %xmm2 +; NEHALEM-NEXT: movups %xmm2, (%rdi) + entry: tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %buf, i8* blockaddress(@foo, %out), i64 22, i32 1, i1 false) br label %out @@ -8,12 +30,6 @@ entry: out: ; preds = %entry %add = fadd float %a, %b ret float %add -; CHECK: foo -; CHECK: movw .L{{.*}}+20(%rip), %{{.*}} -; CHECK: movl .L{{.*}}+16(%rip), %{{.*}} -; CHECK: movq .L{{.*}}+8(%rip), %{{.*}} -; CHECK: movq .L{{.*}}(%rip), %{{.*}} -; CHECK: ret } declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1) nounwind diff --git a/test/CodeGen/X86/pr13577.ll b/test/CodeGen/X86/pr13577.ll index a6b721a7a6f1..2228fbbaa53b 100644 --- a/test/CodeGen/X86/pr13577.ll +++ b/test/CodeGen/X86/pr13577.ll @@ -6,10 +6,7 @@ ; CHECK-NEXT: .long 2139095040 ; CHECK-LABEL: foo: -; CHECK: movq {{.*}}, %rax -; CHECK: shlq $48, %rax -; CHECK: sets %al -; CHECK: testb %al, %al +; CHECK: testb $-128, -15(%rsp) ; CHECK: flds LCPI0_0(%rip) ; CHECK: flds LCPI0_1(%rip) ; CHECK: fcmovne %st(1), %st(0) diff --git a/test/CodeGen/X86/pr15267.ll b/test/CodeGen/X86/pr15267.ll index 95d7deb34170..9fc754aa1128 100644 --- a/test/CodeGen/X86/pr15267.ll +++ b/test/CodeGen/X86/pr15267.ll @@ -1,138 +1,146 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-pc-linux -mcpu=corei7-avx | FileCheck %s define <4 x i3> @test1(<4 x i3>* %in) nounwind { +; CHECK-LABEL: test1: +; CHECK: # BB#0: +; CHECK-NEXT: movzwl (%rdi), %eax +; CHECK-NEXT: movl %eax, %ecx +; CHECK-NEXT: shrl $3, %ecx +; CHECK-NEXT: andl $7, %ecx +; CHECK-NEXT: movl %eax, %edx +; CHECK-NEXT: andl $7, %edx +; CHECK-NEXT: vmovd %edx, %xmm0 +; CHECK-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: movl %eax, %ecx +; CHECK-NEXT: shrl $6, %ecx +; CHECK-NEXT: andl $7, %ecx +; CHECK-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: shrl $9, %eax +; CHECK-NEXT: andl $7, %eax +; CHECK-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0 +; CHECK-NEXT: retq %ret = load <4 x i3>, <4 x i3>* %in, align 1 ret <4 x i3> %ret } -; CHECK-LABEL: test1 -; CHECK: movzwl -; CHECK: shrl $3 -; CHECK: andl $7 -; CHECK: andl $7 -; CHECK: vmovd -; CHECK: pinsrd $1 -; CHECK: shrl $6 -; CHECK: andl $7 -; CHECK: pinsrd $2 -; CHECK: shrl $9 -; CHECK: andl $7 -; CHECK: pinsrd $3 -; CHECK: ret define <4 x i1> @test2(<4 x i1>* %in) nounwind { +; CHECK-LABEL: test2: +; CHECK: # BB#0: +; CHECK-NEXT: movzbl (%rdi), %eax +; CHECK-NEXT: movl %eax, %ecx +; CHECK-NEXT: shrl %ecx +; CHECK-NEXT: andl $1, %ecx +; CHECK-NEXT: movl %eax, %edx +; CHECK-NEXT: andl $1, %edx +; CHECK-NEXT: vmovd %edx, %xmm0 +; CHECK-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: movl %eax, %ecx +; CHECK-NEXT: shrl $2, %ecx +; CHECK-NEXT: andl $1, %ecx +; CHECK-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: shrl $3, %eax +; CHECK-NEXT: andl $1, %eax +; CHECK-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0 +; CHECK-NEXT: retq %ret = load <4 x i1>, <4 x i1>* %in, align 1 ret <4 x i1> %ret } -; CHECK-LABEL: test2 -; CHECK: movzbl -; CHECK: shrl -; CHECK: andl $1 -; CHECK: andl $1 -; CHECK: vmovd -; CHECK: pinsrd $1 -; CHECK: shrl $2 -; CHECK: andl $1 -; CHECK: pinsrd $2 -; CHECK: shrl $3 -; CHECK: andl $1 -; CHECK: pinsrd $3 -; CHECK: ret - define <4 x i64> @test3(<4 x i1>* %in) nounwind { +; CHECK-LABEL: test3: +; CHECK: # BB#0: +; CHECK-NEXT: movzbl (%rdi), %eax +; CHECK-NEXT: movq %rax, %rcx +; CHECK-NEXT: shlq $62, %rcx +; CHECK-NEXT: sarq $63, %rcx +; CHECK-NEXT: movq %rax, %rdx +; CHECK-NEXT: shlq $63, %rdx +; CHECK-NEXT: sarq $63, %rdx +; CHECK-NEXT: vmovd %edx, %xmm0 +; CHECK-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: movq %rax, %rcx +; CHECK-NEXT: shlq $61, %rcx +; CHECK-NEXT: sarq $63, %rcx +; CHECK-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: shlq $60, %rax +; CHECK-NEXT: sarq $63, %rax +; CHECK-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0 +; CHECK-NEXT: vpmovsxdq %xmm0, %xmm1 +; CHECK-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; CHECK-NEXT: vpmovsxdq %xmm0, %xmm0 +; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; CHECK-NEXT: retq %wide.load35 = load <4 x i1>, <4 x i1>* %in, align 1 %sext = sext <4 x i1> %wide.load35 to <4 x i64> ret <4 x i64> %sext } -; CHECK-LABEL: test3 -; CHECK: movzbl -; CHECK: movq -; CHECK: shlq -; CHECK: sarq -; CHECK: movq -; CHECK: shlq -; CHECK: sarq -; CHECK: vmovd -; CHECK: vpinsrd -; CHECK: movq -; CHECK: shlq -; CHECK: sarq -; CHECK: vpinsrd -; CHECK: shlq -; CHECK: sarq -; CHECK: vpinsrd -; CHECK: vpmovsxdq -; CHECK: vmovd -; CHECK: vpinsrd -; CHECK: vpmovsxdq -; CHECK: vinsertf128 -; CHECK: ret - define <16 x i4> @test4(<16 x i4>* %in) nounwind { +; CHECK-LABEL: test4: +; CHECK: # BB#0: +; CHECK-NEXT: movq (%rdi), %rax +; CHECK-NEXT: movl %eax, %ecx +; CHECK-NEXT: shrl $4, %ecx +; CHECK-NEXT: andl $15, %ecx +; CHECK-NEXT: movl %eax, %edx +; CHECK-NEXT: andl $15, %edx +; CHECK-NEXT: vmovd %edx, %xmm0 +; CHECK-NEXT: vpinsrb $1, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: movl %eax, %ecx +; CHECK-NEXT: shrl $8, %ecx +; CHECK-NEXT: andl $15, %ecx +; CHECK-NEXT: vpinsrb $2, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: movl %eax, %ecx +; CHECK-NEXT: shrl $12, %ecx +; CHECK-NEXT: andl $15, %ecx +; CHECK-NEXT: vpinsrb $3, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: movl %eax, %ecx +; CHECK-NEXT: shrl $16, %ecx +; CHECK-NEXT: andl $15, %ecx +; CHECK-NEXT: vpinsrb $4, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: movl %eax, %ecx +; CHECK-NEXT: shrl $20, %ecx +; CHECK-NEXT: andl $15, %ecx +; CHECK-NEXT: vpinsrb $5, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: movl %eax, %ecx +; CHECK-NEXT: shrl $24, %ecx +; CHECK-NEXT: andl $15, %ecx +; CHECK-NEXT: vpinsrb $6, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: movl %eax, %ecx +; CHECK-NEXT: shrl $28, %ecx +; CHECK-NEXT: vpinsrb $7, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: movq %rax, %rcx +; CHECK-NEXT: shrq $32, %rcx +; CHECK-NEXT: andl $15, %ecx +; CHECK-NEXT: vpinsrb $8, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: movq %rax, %rcx +; CHECK-NEXT: shrq $36, %rcx +; CHECK-NEXT: andl $15, %ecx +; CHECK-NEXT: vpinsrb $9, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: movq %rax, %rcx +; CHECK-NEXT: shrq $40, %rcx +; CHECK-NEXT: andl $15, %ecx +; CHECK-NEXT: vpinsrb $10, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: movq %rax, %rcx +; CHECK-NEXT: shrq $44, %rcx +; CHECK-NEXT: andl $15, %ecx +; CHECK-NEXT: vpinsrb $11, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: movq %rax, %rcx +; CHECK-NEXT: shrq $48, %rcx +; CHECK-NEXT: andl $15, %ecx +; CHECK-NEXT: vpinsrb $12, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: movq %rax, %rcx +; CHECK-NEXT: shrq $52, %rcx +; CHECK-NEXT: andl $15, %ecx +; CHECK-NEXT: vpinsrb $13, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: movq %rax, %rcx +; CHECK-NEXT: shrq $56, %rcx +; CHECK-NEXT: andl $15, %ecx +; CHECK-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0 +; CHECK-NEXT: shrq $60, %rax +; CHECK-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0 +; CHECK-NEXT: retq %ret = load <16 x i4>, <16 x i4>* %in, align 1 ret <16 x i4> %ret } - -; CHECK-LABEL: test4 -; CHECK: movl -; CHECK-NEXT: shrl -; CHECK-NEXT: andl -; CHECK-NEXT: movl -; CHECK-NEXT: andl -; CHECK-NEXT: vmovd -; CHECK-NEXT: vpinsrb -; CHECK-NEXT: movl -; CHECK-NEXT: shrl -; CHECK-NEXT: andl -; CHECK-NEXT: vpinsrb -; CHECK-NEXT: movl -; CHECK-NEXT: shrl -; CHECK-NEXT: andl -; CHECK-NEXT: vpinsrb -; CHECK-NEXT: movl -; CHECK-NEXT: shrl -; CHECK-NEXT: andl -; CHECK-NEXT: vpinsrb -; CHECK-NEXT: movl -; CHECK-NEXT: shrl -; CHECK-NEXT: andl -; CHECK-NEXT: vpinsrb -; CHECK-NEXT: movl -; CHECK-NEXT: shrl -; CHECK-NEXT: andl -; CHECK-NEXT: vpinsrb -; CHECK-NEXT: movl -; CHECK-NEXT: shrl -; CHECK-NEXT: vpinsrb -; CHECK-NEXT: movq -; CHECK-NEXT: shrq -; CHECK-NEXT: andl -; CHECK-NEXT: vpinsrb -; CHECK-NEXT: movq -; CHECK-NEXT: shrq -; CHECK-NEXT: andl -; CHECK-NEXT: vpinsrb -; CHECK-NEXT: movq -; CHECK-NEXT: shrq -; CHECK-NEXT: andl -; CHECK-NEXT: vpinsrb -; CHECK-NEXT: movq -; CHECK-NEXT: shrq -; CHECK-NEXT: andl -; CHECK-NEXT: vpinsrb -; CHECK-NEXT: movq -; CHECK-NEXT: shrq -; CHECK-NEXT: andl -; CHECK-NEXT: vpinsrb -; CHECK-NEXT: movq -; CHECK-NEXT: shrq -; CHECK-NEXT: andl -; CHECK-NEXT: vpinsrb -; CHECK-NEXT: movq -; CHECK-NEXT: shrq -; CHECK-NEXT: andl -; CHECK-NEXT: vpinsrb -; CHECK-NEXT: shrq -; CHECK-NEXT: vpinsrb -; CHECK-NEXT: retq diff --git a/test/CodeGen/X86/pr17631.ll b/test/CodeGen/X86/pr17631.ll index 98f951f1b10c..08c393d29d69 100644 --- a/test/CodeGen/X86/pr17631.ll +++ b/test/CodeGen/X86/pr17631.ll @@ -30,5 +30,5 @@ define <8 x float> @foo(<8 x float> %y, i64* %p, double %x) { ; CHECK: foo ; CHECK-NOT: vzeroupper -; CHECK: _ftol2 +; CHECK: {{cvtt|fist}} ; CHECK: ret diff --git a/test/CodeGen/X86/pr21529.ll b/test/CodeGen/X86/pr21529.ll deleted file mode 100644 index 655bc844f503..000000000000 --- a/test/CodeGen/X86/pr21529.ll +++ /dev/null @@ -1,15 +0,0 @@ -; RUN: llc -show-mc-encoding < %s | FileCheck %s - -; Test that the direct object emission selects the and variant with 8 bit -; immediate. -; We used to get this wrong when using direct object emission, but not when -; reading assembly. - -; CHECK: andq $-32, %rsp # encoding: [0x48,0x83,0xe4,0xe0] - -target triple = "x86_64-pc-linux" - -define void @f() { - %foo = alloca i8, align 32 - ret void -} diff --git a/test/CodeGen/X86/pr22019.ll b/test/CodeGen/X86/pr22019.ll index 4cee5d704d3a..cfc53cb6be0b 100644 --- a/test/CodeGen/X86/pr22019.ll +++ b/test/CodeGen/X86/pr22019.ll @@ -20,4 +20,4 @@ define void @pselect() { @var = global i32 0 ; CHECK: alias = var -@alias = alias i32* @var +@alias = alias i32, i32* @var diff --git a/test/CodeGen/X86/pr23900.ll b/test/CodeGen/X86/pr23900.ll deleted file mode 100644 index cbc77161c042..000000000000 --- a/test/CodeGen/X86/pr23900.ll +++ /dev/null @@ -1,29 +0,0 @@ -; RUN: llc -filetype=obj %s -o %t.o -; RUN: llvm-nm %t.o | FileCheck %s - -; Test that it doesn't crash (and produces an object file). -; This use to pass a symbol with a null name to code that expected a valid -; C string. - -; CHECK: U __CxxFrameHandler3 -; CHECK: T f -; CHECK: t f.cleanup -; CHECK: U g -; CHECK: U h - - -target triple = "x86_64-pc-windows-msvc18.0.0" -define void @f(i32 %x) personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) { - invoke void @h() - to label %invoke.cont unwind label %lpad -invoke.cont: - ret void -lpad: - landingpad { i8*, i32 } - cleanup - call void @g(i32 %x) - ret void -} -declare void @h() -declare i32 @__CxxFrameHandler3(...) -declare void @g(i32 %x) diff --git a/test/CodeGen/X86/pr24139.ll b/test/CodeGen/X86/pr24139.ll new file mode 100644 index 000000000000..fbe55abcbf7c --- /dev/null +++ b/test/CodeGen/X86/pr24139.ll @@ -0,0 +1,148 @@ +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s + +; Check that we do not get excessive spilling from splitting of constant live ranges. + +; CHECK-LABEL: PR24139: +; CHECK: # 16-byte Spill +; CHECK-NOT: # 16-byte Spill +; CHECK: retq + +define <2 x double> @PR24139(<2 x double> %arg, <2 x double> %arg1, <2 x double> %arg2) { + %tmp = bitcast <2 x double> %arg to <4 x float> + %tmp3 = fmul <4 x float> %tmp, + %tmp4 = bitcast <2 x double> %arg to <4 x i32> + %tmp5 = and <4 x i32> %tmp4, + %tmp6 = or <4 x i32> %tmp5, + %tmp7 = bitcast <4 x i32> %tmp6 to <4 x float> + %tmp8 = fadd <4 x float> %tmp3, %tmp7 + %tmp9 = tail call <4 x i32> @llvm.x86.sse2.cvttps2dq(<4 x float> %tmp8) #2 + %tmp10 = bitcast <4 x i32> %tmp9 to <2 x i64> + %tmp11 = tail call <4 x float> @llvm.x86.sse2.cvtdq2ps(<4 x i32> %tmp9) #2 + %tmp12 = fmul <4 x float> %tmp11, + %tmp13 = fsub <4 x float> %tmp, %tmp12 + %tmp14 = fmul <4 x float> %tmp11, + %tmp15 = fsub <4 x float> %tmp13, %tmp14 + %tmp16 = fmul <4 x float> %tmp15, %tmp15 + %tmp17 = fmul <4 x float> %tmp15, %tmp16 + %tmp18 = fmul <4 x float> %tmp16, + %tmp19 = fadd <4 x float> %tmp18, + %tmp20 = fmul <4 x float> %tmp16, + %tmp21 = fadd <4 x float> %tmp20, + %tmp22 = fmul <4 x float> %tmp16, %tmp19 + %tmp23 = fadd <4 x float> %tmp22, + %tmp24 = fmul <4 x float> %tmp16, %tmp21 + %tmp25 = fadd <4 x float> %tmp24, + %tmp26 = fmul <4 x float> %tmp16, %tmp23 + %tmp27 = fadd <4 x float> %tmp26, + %tmp28 = fmul <4 x float> %tmp17, %tmp25 + %tmp29 = fadd <4 x float> %tmp15, %tmp28 + %tmp30 = and <2 x i64> %tmp10, + %tmp31 = bitcast <2 x i64> %tmp30 to <4 x i32> + %tmp32 = icmp eq <4 x i32> %tmp31, zeroinitializer + %tmp33 = sext <4 x i1> %tmp32 to <4 x i32> + %tmp34 = bitcast <4 x i32> %tmp33 to <4 x float> + %tmp35 = tail call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %tmp27, <4 x float> %tmp29, <4 x float> %tmp34) #2 + %tmp36 = and <2 x i64> %tmp10, + %tmp37 = bitcast <2 x i64> %tmp36 to <4 x i32> + %tmp38 = icmp eq <4 x i32> %tmp37, zeroinitializer + %tmp39 = sext <4 x i1> %tmp38 to <4 x i32> + %tmp40 = bitcast <4 x float> %tmp35 to <4 x i32> + %tmp41 = xor <4 x i32> %tmp40, + %tmp42 = bitcast <4 x i32> %tmp41 to <4 x float> + %tmp43 = bitcast <4 x i32> %tmp39 to <4 x float> + %tmp44 = tail call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %tmp42, <4 x float> %tmp35, <4 x float> %tmp43) #2 + %tmp45 = bitcast <2 x double> %arg1 to <4 x float> + %tmp46 = fmul <4 x float> %tmp45, + %tmp47 = bitcast <2 x double> %arg1 to <4 x i32> + %tmp48 = and <4 x i32> %tmp47, + %tmp49 = or <4 x i32> %tmp48, + %tmp50 = bitcast <4 x i32> %tmp49 to <4 x float> + %tmp51 = fadd <4 x float> %tmp46, %tmp50 + %tmp52 = tail call <4 x i32> @llvm.x86.sse2.cvttps2dq(<4 x float> %tmp51) #2 + %tmp53 = bitcast <4 x i32> %tmp52 to <2 x i64> + %tmp54 = tail call <4 x float> @llvm.x86.sse2.cvtdq2ps(<4 x i32> %tmp52) #2 + %tmp55 = fmul <4 x float> %tmp54, + %tmp56 = fsub <4 x float> %tmp45, %tmp55 + %tmp57 = fmul <4 x float> %tmp54, + %tmp58 = fsub <4 x float> %tmp56, %tmp57 + %tmp59 = fmul <4 x float> %tmp58, %tmp58 + %tmp60 = fmul <4 x float> %tmp58, %tmp59 + %tmp61 = fmul <4 x float> %tmp59, + %tmp62 = fadd <4 x float> %tmp61, + %tmp63 = fmul <4 x float> %tmp59, + %tmp64 = fadd <4 x float> %tmp63, + %tmp65 = fmul <4 x float> %tmp59, %tmp62 + %tmp66 = fadd <4 x float> %tmp65, + %tmp67 = fmul <4 x float> %tmp59, %tmp64 + %tmp68 = fadd <4 x float> %tmp67, + %tmp69 = fmul <4 x float> %tmp59, %tmp66 + %tmp70 = fadd <4 x float> %tmp69, + %tmp71 = fmul <4 x float> %tmp60, %tmp68 + %tmp72 = fadd <4 x float> %tmp58, %tmp71 + %tmp73 = and <2 x i64> %tmp53, + %tmp74 = bitcast <2 x i64> %tmp73 to <4 x i32> + %tmp75 = icmp eq <4 x i32> %tmp74, zeroinitializer + %tmp76 = sext <4 x i1> %tmp75 to <4 x i32> + %tmp77 = bitcast <4 x i32> %tmp76 to <4 x float> + %tmp78 = tail call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %tmp70, <4 x float> %tmp72, <4 x float> %tmp77) #2 + %tmp79 = and <2 x i64> %tmp53, + %tmp80 = bitcast <2 x i64> %tmp79 to <4 x i32> + %tmp81 = icmp eq <4 x i32> %tmp80, zeroinitializer + %tmp82 = sext <4 x i1> %tmp81 to <4 x i32> + %tmp83 = bitcast <4 x float> %tmp78 to <4 x i32> + %tmp84 = xor <4 x i32> %tmp83, + %tmp85 = bitcast <4 x i32> %tmp84 to <4 x float> + %tmp86 = bitcast <4 x i32> %tmp82 to <4 x float> + %tmp87 = tail call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %tmp85, <4 x float> %tmp78, <4 x float> %tmp86) #2 + %tmp88 = fadd <4 x float> %tmp44, %tmp87 + %tmp89 = bitcast <2 x double> %arg2 to <4 x float> + %tmp90 = fmul <4 x float> %tmp89, + %tmp91 = bitcast <2 x double> %arg2 to <4 x i32> + %tmp92 = and <4 x i32> %tmp91, + %tmp93 = or <4 x i32> %tmp92, + %tmp94 = bitcast <4 x i32> %tmp93 to <4 x float> + %tmp95 = fadd <4 x float> %tmp90, %tmp94 + %tmp96 = tail call <4 x i32> @llvm.x86.sse2.cvttps2dq(<4 x float> %tmp95) #2 + %tmp97 = bitcast <4 x i32> %tmp96 to <2 x i64> + %tmp98 = tail call <4 x float> @llvm.x86.sse2.cvtdq2ps(<4 x i32> %tmp96) #2 + %tmp99 = fmul <4 x float> %tmp98, + %tmp100 = fsub <4 x float> %tmp89, %tmp99 + %tmp101 = fmul <4 x float> %tmp98, + %tmp102 = fsub <4 x float> %tmp100, %tmp101 + %tmp103 = fmul <4 x float> %tmp102, %tmp102 + %tmp104 = fmul <4 x float> %tmp102, %tmp103 + %tmp105 = fmul <4 x float> %tmp103, + %tmp106 = fadd <4 x float> %tmp105, + %tmp107 = fmul <4 x float> %tmp103, + %tmp108 = fadd <4 x float> %tmp107, + %tmp109 = fmul <4 x float> %tmp103, %tmp106 + %tmp110 = fadd <4 x float> %tmp109, + %tmp111 = fmul <4 x float> %tmp103, %tmp108 + %tmp112 = fadd <4 x float> %tmp111, + %tmp113 = fmul <4 x float> %tmp103, %tmp110 + %tmp114 = fadd <4 x float> %tmp113, + %tmp115 = fmul <4 x float> %tmp104, %tmp112 + %tmp116 = fadd <4 x float> %tmp102, %tmp115 + %tmp117 = and <2 x i64> %tmp97, + %tmp118 = bitcast <2 x i64> %tmp117 to <4 x i32> + %tmp119 = icmp eq <4 x i32> %tmp118, zeroinitializer + %tmp120 = sext <4 x i1> %tmp119 to <4 x i32> + %tmp121 = bitcast <4 x i32> %tmp120 to <4 x float> + %tmp122 = tail call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %tmp114, <4 x float> %tmp116, <4 x float> %tmp121) #2 + %tmp123 = and <2 x i64> %tmp97, + %tmp124 = bitcast <2 x i64> %tmp123 to <4 x i32> + %tmp125 = icmp eq <4 x i32> %tmp124, zeroinitializer + %tmp126 = sext <4 x i1> %tmp125 to <4 x i32> + %tmp127 = bitcast <4 x float> %tmp122 to <4 x i32> + %tmp128 = xor <4 x i32> %tmp127, + %tmp129 = bitcast <4 x i32> %tmp128 to <4 x float> + %tmp130 = bitcast <4 x i32> %tmp126 to <4 x float> + %tmp131 = tail call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %tmp129, <4 x float> %tmp122, <4 x float> %tmp130) #2 + %tmp132 = fadd <4 x float> %tmp88, %tmp131 + %tmp133 = bitcast <4 x float> %tmp132 to <2 x double> + ret <2 x double> %tmp133 +} + +declare <4 x i32> @llvm.x86.sse2.cvttps2dq(<4 x float>) +declare <4 x float> @llvm.x86.sse2.cvtdq2ps(<4 x i32>) +declare <4 x float> @llvm.x86.sse41.blendvps(<4 x float>, <4 x float>, <4 x float>) diff --git a/test/CodeGen/X86/pr24602.ll b/test/CodeGen/X86/pr24602.ll new file mode 100644 index 000000000000..9c029aeefec9 --- /dev/null +++ b/test/CodeGen/X86/pr24602.ll @@ -0,0 +1,17 @@ +; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s + +; PR24602: Make sure we don't barf on non-foldable code (with opaque constants). + +; CHECK-LABEL: pr24602: +; CHECK-NEXT: # BB#0 +; CHECK-NEXT: movabsq $-10000000000, [[CST:%[a-z0-9]+]] +; CHECK-NEXT: imulq [[CST]], %rsi +; CHECK-NEXT: leaq (%rdi,%rsi,8), %rax +; CHECK-NEXT: movq [[CST]], (%rdi,%rsi,8) +; CHECK-NEXT: retq +define i64* @pr24602(i64* %p, i64 %n) nounwind { + %mul = mul nsw i64 %n, -10000000000 + %add.ptr = getelementptr inbounds i64, i64* %p, i64 %mul + store i64 -10000000000, i64* %add.ptr + ret i64* %add.ptr +} diff --git a/test/CodeGen/X86/pr25828.ll b/test/CodeGen/X86/pr25828.ll new file mode 100644 index 000000000000..8fbabc7d0c6d --- /dev/null +++ b/test/CodeGen/X86/pr25828.ll @@ -0,0 +1,30 @@ +; RUN: llc < %s -mtriple=i686-pc-windows-msvc -relocation-model=pic | FileCheck %s +; MOVPC32r should not generate CFI under windows + +; CHECK-LABEL: _foo: +; CHECK-NOT: .cfi_adjust_cfa_offset +define void @foo(i8) { +entry-block: + switch i8 %0, label %bb2 [ + i8 1, label %bb1 + i8 2, label %bb2 + i8 3, label %bb3 + i8 4, label %bb4 + i8 5, label %bb5 + ] + +bb1: + ret void + +bb2: + ret void + +bb3: + ret void + +bb4: + ret void + +bb5: + ret void +} diff --git a/test/CodeGen/X86/prolog-push-seq.ll b/test/CodeGen/X86/prolog-push-seq.ll new file mode 100644 index 000000000000..f23791aef922 --- /dev/null +++ b/test/CodeGen/X86/prolog-push-seq.ll @@ -0,0 +1,19 @@ +; RUN: llc < %s | FileCheck %s + +target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32" +target triple = "i386-pc-windows-msvc18.0.0" + +declare x86_thiscallcc void @bar(i32 %a, i32 %b) + +define fastcc void @foo(i32 %a, i32 %b) #0 { +; CHECK-LABEL: foo: +; CHECK: subl $64, %esp +; CHECK-NEXT: pushl +; CHECK-NEXT: calll _bar + %local = alloca i32, i32 16 + call x86_thiscallcc void @bar(i32 %a, i32 %b) + call void asm sideeffect "nop", "~{ax},~{bx},~{cx},~{dx},~{bp},~{si},~{di}"() + ret void +} + +attributes #0 = { nounwind optsize "no-frame-pointer-elim-non-leaf"} \ No newline at end of file diff --git a/test/CodeGen/X86/pseudo_cmov_lower.ll b/test/CodeGen/X86/pseudo_cmov_lower.ll new file mode 100644 index 000000000000..c59e3478ff51 --- /dev/null +++ b/test/CodeGen/X86/pseudo_cmov_lower.ll @@ -0,0 +1,267 @@ +; RUN: llc < %s -mtriple=i386-linux-gnu -o - | FileCheck %s + +; This test checks that only a single js gets generated in the final code +; for lowering the CMOV pseudos that get created for this IR. +; CHECK-LABEL: foo1: +; CHECK: js +; CHECK-NOT: js +define i32 @foo1(i32 %v1, i32 %v2, i32 %v3) nounwind { +entry: + %cmp = icmp slt i32 %v1, 0 + %v2.v3 = select i1 %cmp, i32 %v2, i32 %v3 + %v1.v2 = select i1 %cmp, i32 %v1, i32 %v2 + %sub = sub i32 %v1.v2, %v2.v3 + ret i32 %sub +} + +; This test checks that only a single js gets generated in the final code +; for lowering the CMOV pseudos that get created for this IR. This makes +; sure the code for the lowering for opposite conditions gets tested. +; CHECK-LABEL: foo11: +; CHECK: js +; CHECK-NOT: js +; CHECK-NOT: jns +define i32 @foo11(i32 %v1, i32 %v2, i32 %v3) nounwind { +entry: + %cmp1 = icmp slt i32 %v1, 0 + %v2.v3 = select i1 %cmp1, i32 %v2, i32 %v3 + %cmp2 = icmp sge i32 %v1, 0 + %v1.v2 = select i1 %cmp2, i32 %v1, i32 %v2 + %sub = sub i32 %v1.v2, %v2.v3 + ret i32 %sub +} + +; This test checks that only a single js gets generated in the final code +; for lowering the CMOV pseudos that get created for this IR. +; CHECK-LABEL: foo2: +; CHECK: js +; CHECK-NOT: js +define i32 @foo2(i8 %v1, i8 %v2, i8 %v3) nounwind { +entry: + %cmp = icmp slt i8 %v1, 0 + %v2.v3 = select i1 %cmp, i8 %v2, i8 %v3 + %v1.v2 = select i1 %cmp, i8 %v1, i8 %v2 + %t1 = sext i8 %v2.v3 to i32 + %t2 = sext i8 %v1.v2 to i32 + %sub = sub i32 %t1, %t2 + ret i32 %sub +} + +; This test checks that only a single js gets generated in the final code +; for lowering the CMOV pseudos that get created for this IR. +; CHECK-LABEL: foo3: +; CHECK: js +; CHECK-NOT: js +define i32 @foo3(i16 %v1, i16 %v2, i16 %v3) nounwind { +entry: + %cmp = icmp slt i16 %v1, 0 + %v2.v3 = select i1 %cmp, i16 %v2, i16 %v3 + %v1.v2 = select i1 %cmp, i16 %v1, i16 %v2 + %t1 = sext i16 %v2.v3 to i32 + %t2 = sext i16 %v1.v2 to i32 + %sub = sub i32 %t1, %t2 + ret i32 %sub +} + +; This test checks that only a single js gets generated in the final code +; for lowering the CMOV pseudos that get created for this IR. +; CHECK-LABEL: foo4: +; CHECK: js +; CHECK-NOT: js +define float @foo4(i32 %v1, float %v2, float %v3, float %v4) nounwind { +entry: + %cmp = icmp slt i32 %v1, 0 + %t1 = select i1 %cmp, float %v2, float %v3 + %t2 = select i1 %cmp, float %v3, float %v4 + %sub = fsub float %t1, %t2 + ret float %sub +} + +; This test checks that only a single je gets generated in the final code +; for lowering the CMOV pseudos that get created for this IR. +; CHECK-LABEL: foo5: +; CHECK: je +; CHECK-NOT: je +define double @foo5(i32 %v1, double %v2, double %v3, double %v4) nounwind { +entry: + %cmp = icmp eq i32 %v1, 0 + %t1 = select i1 %cmp, double %v2, double %v3 + %t2 = select i1 %cmp, double %v3, double %v4 + %sub = fsub double %t1, %t2 + ret double %sub +} + +; This test checks that only a single je gets generated in the final code +; for lowering the CMOV pseudos that get created for this IR. +; CHECK-LABEL: foo6: +; CHECK: je +; CHECK-NOT: je +define <4 x float> @foo6(i32 %v1, <4 x float> %v2, <4 x float> %v3, <4 x float> %v4) nounwind { +entry: + %cmp = icmp eq i32 %v1, 0 + %t1 = select i1 %cmp, <4 x float> %v2, <4 x float> %v3 + %t2 = select i1 %cmp, <4 x float> %v3, <4 x float> %v4 + %sub = fsub <4 x float> %t1, %t2 + ret <4 x float> %sub +} + +; This test checks that only a single je gets generated in the final code +; for lowering the CMOV pseudos that get created for this IR. +; CHECK-LABEL: foo7: +; CHECK: je +; CHECK-NOT: je +define <2 x double> @foo7(i32 %v1, <2 x double> %v2, <2 x double> %v3, <2 x double> %v4) nounwind { +entry: + %cmp = icmp eq i32 %v1, 0 + %t1 = select i1 %cmp, <2 x double> %v2, <2 x double> %v3 + %t2 = select i1 %cmp, <2 x double> %v3, <2 x double> %v4 + %sub = fsub <2 x double> %t1, %t2 + ret <2 x double> %sub +} + +; This test checks that only a single ja gets generated in the final code +; for lowering the CMOV pseudos that get created for this IR. This combines +; all the supported types together into one long string of selects based +; on the same condition. +; CHECK-LABEL: foo8: +; CHECK: ja +; CHECK-NOT: ja +define void @foo8(i32 %v1, + i8 %v2, i8 %v3, + i16 %v12, i16 %v13, + i32 %v22, i32 %v23, + float %v32, float %v33, + double %v42, double %v43, + <4 x float> %v52, <4 x float> %v53, + <2 x double> %v62, <2 x double> %v63, + <8 x float> %v72, <8 x float> %v73, + <4 x double> %v82, <4 x double> %v83, + <16 x float> %v92, <16 x float> %v93, + <8 x double> %v102, <8 x double> %v103, + i8 * %dst) nounwind { +entry: + %add.ptr11 = getelementptr inbounds i8, i8* %dst, i32 2 + %a11 = bitcast i8* %add.ptr11 to i16* + + %add.ptr21 = getelementptr inbounds i8, i8* %dst, i32 4 + %a21 = bitcast i8* %add.ptr21 to i32* + + %add.ptr31 = getelementptr inbounds i8, i8* %dst, i32 8 + %a31 = bitcast i8* %add.ptr31 to float* + + %add.ptr41 = getelementptr inbounds i8, i8* %dst, i32 16 + %a41 = bitcast i8* %add.ptr41 to double* + + %add.ptr51 = getelementptr inbounds i8, i8* %dst, i32 32 + %a51 = bitcast i8* %add.ptr51 to <4 x float>* + + %add.ptr61 = getelementptr inbounds i8, i8* %dst, i32 48 + %a61 = bitcast i8* %add.ptr61 to <2 x double>* + + %add.ptr71 = getelementptr inbounds i8, i8* %dst, i32 64 + %a71 = bitcast i8* %add.ptr71 to <8 x float>* + + %add.ptr81 = getelementptr inbounds i8, i8* %dst, i32 128 + %a81 = bitcast i8* %add.ptr81 to <4 x double>* + + %add.ptr91 = getelementptr inbounds i8, i8* %dst, i32 64 + %a91 = bitcast i8* %add.ptr91 to <16 x float>* + + %add.ptr101 = getelementptr inbounds i8, i8* %dst, i32 128 + %a101 = bitcast i8* %add.ptr101 to <8 x double>* + + ; These operations are necessary, because select of two single use loads + ; ends up getting optimized into a select of two leas, followed by a + ; single load of the selected address. + %t13 = xor i16 %v13, 11 + %t23 = xor i32 %v23, 1234 + %t33 = fadd float %v33, %v32 + %t43 = fadd double %v43, %v42 + %t53 = fadd <4 x float> %v53, %v52 + %t63 = fadd <2 x double> %v63, %v62 + %t73 = fsub <8 x float> %v73, %v72 + %t83 = fsub <4 x double> %v83, %v82 + %t93 = fsub <16 x float> %v93, %v92 + %t103 = fsub <8 x double> %v103, %v102 + + %cmp = icmp ugt i32 %v1, 31 + %t11 = select i1 %cmp, i16 %v12, i16 %t13 + %t21 = select i1 %cmp, i32 %v22, i32 %t23 + %t31 = select i1 %cmp, float %v32, float %t33 + %t41 = select i1 %cmp, double %v42, double %t43 + %t51 = select i1 %cmp, <4 x float> %v52, <4 x float> %t53 + %t61 = select i1 %cmp, <2 x double> %v62, <2 x double> %t63 + %t71 = select i1 %cmp, <8 x float> %v72, <8 x float> %t73 + %t81 = select i1 %cmp, <4 x double> %v82, <4 x double> %t83 + %t91 = select i1 %cmp, <16 x float> %v92, <16 x float> %t93 + %t101 = select i1 %cmp, <8 x double> %v102, <8 x double> %t103 + + store i16 %t11, i16* %a11, align 2 + store i32 %t21, i32* %a21, align 4 + store float %t31, float* %a31, align 4 + store double %t41, double* %a41, align 8 + store <4 x float> %t51, <4 x float>* %a51, align 16 + store <2 x double> %t61, <2 x double>* %a61, align 16 + store <8 x float> %t71, <8 x float>* %a71, align 32 + store <4 x double> %t81, <4 x double>* %a81, align 32 + store <16 x float> %t91, <16 x float>* %a91, align 32 + store <8 x double> %t101, <8 x double>* %a101, align 32 + + ret void +} + +; This test checks that only a single ja gets generated in the final code +; for lowering the CMOV pseudos that get created for this IR. +; on the same condition. +; Contrary to my expectations, this doesn't exercise the code for +; CMOV_V8I1, CMOV_V16I1, CMOV_V32I1, or CMOV_V64I1. Instead the selects all +; get lowered into vector length number of selects, which all eventually turn +; into a huge number of CMOV_GR8, which are all contiguous, so the optimization +; kicks in as long as CMOV_GR8 is supported. I couldn't find a way to get +; CMOV_V*I1 pseudo-opcodes to get generated. If a way exists to get CMOV_V*1 +; pseudo-opcodes to be generated, this test should be replaced with one that +; tests those opcodes. +; +; CHECK-LABEL: foo9: +; CHECK: ja +; CHECK-NOT: ja +define void @foo9(i32 %v1, + <8 x i1> %v12, <8 x i1> %v13, + <16 x i1> %v22, <16 x i1> %v23, + <32 x i1> %v32, <32 x i1> %v33, + <64 x i1> %v42, <64 x i1> %v43, + i8 * %dst) nounwind { +entry: + %add.ptr11 = getelementptr inbounds i8, i8* %dst, i32 0 + %a11 = bitcast i8* %add.ptr11 to <8 x i1>* + + %add.ptr21 = getelementptr inbounds i8, i8* %dst, i32 4 + %a21 = bitcast i8* %add.ptr21 to <16 x i1>* + + %add.ptr31 = getelementptr inbounds i8, i8* %dst, i32 8 + %a31 = bitcast i8* %add.ptr31 to <32 x i1>* + + %add.ptr41 = getelementptr inbounds i8, i8* %dst, i32 16 + %a41 = bitcast i8* %add.ptr41 to <64 x i1>* + + ; These operations are necessary, because select of two single use loads + ; ends up getting optimized into a select of two leas, followed by a + ; single load of the selected address. + %t13 = xor <8 x i1> %v13, %v12 + %t23 = xor <16 x i1> %v23, %v22 + %t33 = xor <32 x i1> %v33, %v32 + %t43 = xor <64 x i1> %v43, %v42 + + %cmp = icmp ugt i32 %v1, 31 + %t11 = select i1 %cmp, <8 x i1> %v12, <8 x i1> %t13 + %t21 = select i1 %cmp, <16 x i1> %v22, <16 x i1> %t23 + %t31 = select i1 %cmp, <32 x i1> %v32, <32 x i1> %t33 + %t41 = select i1 %cmp, <64 x i1> %v42, <64 x i1> %t43 + + store <8 x i1> %t11, <8 x i1>* %a11, align 16 + store <16 x i1> %t21, <16 x i1>* %a21, align 4 + store <32 x i1> %t31, <32 x i1>* %a31, align 8 + store <64 x i1> %t41, <64 x i1>* %a41, align 16 + + ret void +} diff --git a/test/CodeGen/X86/pseudo_cmov_lower1.ll b/test/CodeGen/X86/pseudo_cmov_lower1.ll new file mode 100644 index 000000000000..4ce131bb8645 --- /dev/null +++ b/test/CodeGen/X86/pseudo_cmov_lower1.ll @@ -0,0 +1,39 @@ +; RUN: llc < %s -mtriple=i386-linux-gnu -mattr=+sse2 -o - | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-linux-gnu -o - | FileCheck %s + +; This test checks that only a single jae gets generated in the final code +; for lowering the CMOV pseudos that get created for this IR. +; CHECK-LABEL: foo1: +; CHECK: jae +; CHECK-NOT: jae +define double @foo1(float %p1, double %p2, double %p3) nounwind { +entry: + %c1 = fcmp oge float %p1, 0.000000e+00 + %d0 = fadd double %p2, 1.25e0 + %d1 = fadd double %p3, 1.25e0 + %d2 = select i1 %c1, double %d0, double %d1 + %d3 = select i1 %c1, double %d0, double %p2 + %d4 = select i1 %c1, double %p3, double %d1 + %d5 = fsub double %d2, %d3 + %d6 = fadd double %d5, %d4 + ret double %d6 +} + +; This test checks that only a single jae gets generated in the final code +; for lowering the CMOV pseudos that get created for this IR. +; CHECK-LABEL: foo2: +; CHECK: jae +; CHECK-NOT: jae +define float @foo2(float %p1, float %p2, float %p3) nounwind { +entry: + %c1 = fcmp oge float %p1, 0.000000e+00 + %d0 = fadd float %p2, 1.25e0 + %d1 = fadd float %p3, 1.25e0 + %d2 = select i1 %c1, float %d0, float %d1 + %d3 = select i1 %c1, float %d1, float %p2 + %d4 = select i1 %c1, float %d0, float %p3 + %d5 = fsub float %d2, %d3 + %d6 = fadd float %d5, %d4 + ret float %d6 +} + diff --git a/test/CodeGen/X86/pseudo_cmov_lower2.ll b/test/CodeGen/X86/pseudo_cmov_lower2.ll new file mode 100644 index 000000000000..0133963b36d0 --- /dev/null +++ b/test/CodeGen/X86/pseudo_cmov_lower2.ll @@ -0,0 +1,100 @@ +; RUN: llc < %s -mtriple=x86_64-linux-gnu -o - | FileCheck %s + +; This test checks that only a single jae gets generated in the final code +; for lowering the CMOV pseudos that get created for this IR. The tricky part +; of this test is that it tests the special PHI operand rewriting code in +; X86TargetLowering::EmitLoweredSelect. +; +; CHECK-LABEL: foo1: +; CHECK: jae +; CHECK-NOT: jae +define double @foo1(float %p1, double %p2, double %p3) nounwind { +entry: + %c1 = fcmp oge float %p1, 0.000000e+00 + %d0 = fadd double %p2, 1.25e0 + %d1 = fadd double %p3, 1.25e0 + %d2 = select i1 %c1, double %d0, double %d1 + %d3 = select i1 %c1, double %d2, double %p2 + %d4 = select i1 %c1, double %d3, double %p3 + %d5 = fsub double %d2, %d3 + %d6 = fadd double %d5, %d4 + ret double %d6 +} + +; This test checks that only a single jae gets generated in the final code +; for lowering the CMOV pseudos that get created for this IR. The tricky part +; of this test is that it tests the special PHI operand rewriting code in +; X86TargetLowering::EmitLoweredSelect. +; +; CHECK-LABEL: foo2: +; CHECK: jae +; CHECK-NOT: jae +define double @foo2(float %p1, double %p2, double %p3) nounwind { +entry: + %c1 = fcmp oge float %p1, 0.000000e+00 + %d0 = fadd double %p2, 1.25e0 + %d1 = fadd double %p3, 1.25e0 + %d2 = select i1 %c1, double %d0, double %d1 + %d3 = select i1 %c1, double %p2, double %d2 + %d4 = select i1 %c1, double %p3, double %d3 + %d5 = fsub double %d2, %d3 + %d6 = fadd double %d5, %d4 + ret double %d6 +} + +; This test checks that only a single js gets generated in the final code +; for lowering the CMOV pseudos that get created for this IR. The tricky part +; of this test is that it tests the special PHI operand rewriting code in +; X86TargetLowering::EmitLoweredSelect. It also tests to make sure all +; the operands of the resulting instructions are from the proper places. +; +; CHECK-LABEL: foo3: +; CHECK: js +; CHECK-NOT: js +; CHECK-LABEL: # BB#1: +; CHECK-DAG: movapd %xmm2, %xmm1 +; CHECK-DAG: movapd %xmm2, %xmm0 +; CHECK-LABEL:.LBB2_2: +; CHECK: divsd %xmm1, %xmm0 +; CHECK: ret +define double @foo3(i32 %p1, double %p2, double %p3, + double %p4, double %p5) nounwind { +entry: + %c1 = icmp slt i32 %p1, 0 + %d2 = select i1 %c1, double %p2, double %p3 + %d3 = select i1 %c1, double %p3, double %p4 + %d4 = select i1 %c1, double %d2, double %d3 + %d5 = fdiv double %d4, %d3 + ret double %d5 +} + +; This test checks that only a single js gets generated in the final code +; for lowering the CMOV pseudos that get created for this IR. The tricky part +; of this test is that it tests the special PHI operand rewriting code in +; X86TargetLowering::EmitLoweredSelect. It also tests to make sure all +; the operands of the resulting instructions are from the proper places +; when the "opposite condition" handling code in the compiler is used. +; This should be the same code as foo3 above, because we use the opposite +; condition code in the second two selects, but we also swap the operands +; of the selects to give the same actual computation. +; +; CHECK-LABEL: foo4: +; CHECK: js +; CHECK-NOT: js +; CHECK-LABEL: # BB#1: +; CHECK-DAG: movapd %xmm2, %xmm1 +; CHECK-DAG: movapd %xmm2, %xmm0 +; CHECK-LABEL:.LBB3_2: +; CHECK: divsd %xmm1, %xmm0 +; CHECK: ret +define double @foo4(i32 %p1, double %p2, double %p3, + double %p4, double %p5) nounwind { +entry: + %c1 = icmp slt i32 %p1, 0 + %d2 = select i1 %c1, double %p2, double %p3 + %c2 = icmp sge i32 %p1, 0 + %d3 = select i1 %c2, double %p4, double %p3 + %d4 = select i1 %c2, double %d3, double %d2 + %d5 = fdiv double %d4, %d3 + ret double %d5 +} diff --git a/test/CodeGen/X86/psubus.ll b/test/CodeGen/X86/psubus.ll index 4b83b55997e2..c6d118d6da69 100644 --- a/test/CodeGen/X86/psubus.ll +++ b/test/CodeGen/X86/psubus.ll @@ -1,11 +1,22 @@ -; RUN: llc -mcpu=core2 < %s | FileCheck %s -check-prefix=SSSE3 -; RUN: llc -mcpu=corei7-avx < %s | FileCheck %s -check-prefix=AVX1 -; RUN: llc -mcpu=core-avx2 < %s | FileCheck %s -check-prefix=AVX2 - -target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" -target triple = "x86_64-apple-macosx10.8.0" +; RUN: llc < %s -mtriple=x86_64-apple-macosx10.8.0 -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-apple-macosx10.8.0 -mattr=+ssse3 | FileCheck %s --check-prefix=SSE --check-prefix=SSSE3 +; RUN: llc < %s -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2 define void @test1(i16* nocapture %head) nounwind { +; SSE-LABEL: test1: +; SSE: ## BB#0: ## %vector.ph +; SSE-NEXT: movdqu (%rdi), %xmm0 +; SSE-NEXT: psubusw {{.*}}(%rip), %xmm0 +; SSE-NEXT: movdqu %xmm0, (%rdi) +; SSE-NEXT: retq +; +; AVX-LABEL: test1: +; AVX: ## BB#0: ## %vector.ph +; AVX-NEXT: vmovdqu (%rdi), %xmm0 +; AVX-NEXT: vpsubusw {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vmovdqu %xmm0, (%rdi) +; AVX-NEXT: retq vector.ph: %0 = getelementptr inbounds i16, i16* %head, i64 0 %1 = bitcast i16* %0 to <8 x i16>* @@ -15,30 +26,22 @@ vector.ph: %5 = select <8 x i1> %3, <8 x i16> %4, <8 x i16> zeroinitializer store <8 x i16> %5, <8 x i16>* %1, align 2 ret void - -; SSSE3: @test1 -; SSSE3: # BB#0: -; SSSE3-NEXT: movdqu (%rdi), %xmm0 -; SSSE3-NEXT: psubusw LCPI0_0(%rip), %xmm0 -; SSSE3-NEXT: movdqu %xmm0, (%rdi) -; SSSE3-NEXT: retq - -; AVX1: @test1 -; AVX1: # BB#0: -; AVX1-NEXT: vmovdqu (%rdi), %xmm0 -; AVX1-NEXT: vpsubusw LCPI0_0(%rip), %xmm0, %xmm0 -; AVX1-NEXT: vmovdqu %xmm0, (%rdi) -; AVX1-NEXT: retq - -; AVX2: @test1 -; AVX2: # BB#0: -; AVX2-NEXT: vmovdqu (%rdi), %xmm0 -; AVX2-NEXT: vpsubusw LCPI0_0(%rip), %xmm0, %xmm0 -; AVX2-NEXT: vmovdqu %xmm0, (%rdi) -; AVX2-NEXT: retq } define void @test2(i16* nocapture %head) nounwind { +; SSE-LABEL: test2: +; SSE: ## BB#0: ## %vector.ph +; SSE-NEXT: movdqu (%rdi), %xmm0 +; SSE-NEXT: psubusw {{.*}}(%rip), %xmm0 +; SSE-NEXT: movdqu %xmm0, (%rdi) +; SSE-NEXT: retq +; +; AVX-LABEL: test2: +; AVX: ## BB#0: ## %vector.ph +; AVX-NEXT: vmovdqu (%rdi), %xmm0 +; AVX-NEXT: vpsubusw {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vmovdqu %xmm0, (%rdi) +; AVX-NEXT: retq vector.ph: %0 = getelementptr inbounds i16, i16* %head, i64 0 %1 = bitcast i16* %0 to <8 x i16>* @@ -48,30 +51,46 @@ vector.ph: %5 = select <8 x i1> %3, <8 x i16> %4, <8 x i16> zeroinitializer store <8 x i16> %5, <8 x i16>* %1, align 2 ret void - -; SSSE3: @test2 -; SSSE3: # BB#0: -; SSSE3-NEXT: movdqu (%rdi), %xmm0 -; SSSE3-NEXT: psubusw LCPI1_0(%rip), %xmm0 -; SSSE3-NEXT: movdqu %xmm0, (%rdi) -; SSSE3-NEXT: retq - -; AVX1: @test2 -; AVX1: # BB#0: -; AVX1-NEXT: vmovdqu (%rdi), %xmm0 -; AVX1-NEXT: vpsubusw LCPI1_0(%rip), %xmm0, %xmm0 -; AVX1-NEXT: vmovdqu %xmm0, (%rdi) -; AVX1-NEXT: retq - -; AVX2: @test2 -; AVX2: # BB#0: -; AVX2-NEXT: vmovdqu (%rdi), %xmm0 -; AVX2-NEXT: vpsubusw LCPI1_0(%rip), %xmm0, %xmm0 -; AVX2-NEXT: vmovdqu %xmm0, (%rdi) -; AVX2-NEXT: retq } define void @test3(i16* nocapture %head, i16 zeroext %w) nounwind { +; SSE2-LABEL: test3: +; SSE2: ## BB#0: ## %vector.ph +; SSE2-NEXT: movd %esi, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] +; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4] +; SSE2-NEXT: movdqu (%rdi), %xmm1 +; SSE2-NEXT: psubusw %xmm0, %xmm1 +; SSE2-NEXT: movdqu %xmm1, (%rdi) +; SSE2-NEXT: retq +; +; SSSE3-LABEL: test3: +; SSSE3: ## BB#0: ## %vector.ph +; SSSE3-NEXT: movd %esi, %xmm0 +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1] +; SSSE3-NEXT: movdqu (%rdi), %xmm1 +; SSSE3-NEXT: psubusw %xmm0, %xmm1 +; SSSE3-NEXT: movdqu %xmm1, (%rdi) +; SSSE3-NEXT: retq +; +; AVX1-LABEL: test3: +; AVX1: ## BB#0: ## %vector.ph +; AVX1-NEXT: vmovd %esi, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1] +; AVX1-NEXT: vmovdqu (%rdi), %xmm1 +; AVX1-NEXT: vpsubusw %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vmovdqu %xmm0, (%rdi) +; AVX1-NEXT: retq +; +; AVX2-LABEL: test3: +; AVX2: ## BB#0: ## %vector.ph +; AVX2-NEXT: vmovd %esi, %xmm0 +; AVX2-NEXT: vpbroadcastw %xmm0, %xmm0 +; AVX2-NEXT: vmovdqu (%rdi), %xmm1 +; AVX2-NEXT: vpsubusw %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vmovdqu %xmm0, (%rdi) +; AVX2-NEXT: retq vector.ph: %0 = insertelement <8 x i16> undef, i16 %w, i32 0 %broadcast15 = shufflevector <8 x i16> %0, <8 x i16> undef, <8 x i32> zeroinitializer @@ -83,36 +102,22 @@ vector.ph: %6 = select <8 x i1> %4, <8 x i16> zeroinitializer, <8 x i16> %5 store <8 x i16> %6, <8 x i16>* %2, align 2 ret void - -; SSSE3: @test3 -; SSSE3: # BB#0: -; SSSE3-NEXT: movd %esi, %xmm0 -; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1] -; SSSE3-NEXT: movdqu (%rdi), %xmm1 -; SSSE3-NEXT: psubusw %xmm0, %xmm1 -; SSSE3-NEXT: movdqu %xmm1, (%rdi) -; SSSE3-NEXT: retq - -; AVX1: @test3 -; AVX1: # BB#0: -; AVX1-NEXT: vmovd %esi, %xmm0 -; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1] -; AVX1-NEXT: vmovdqu (%rdi), %xmm1 -; AVX1-NEXT: vpsubusw %xmm0, %xmm1, %xmm0 -; AVX1-NEXT: vmovdqu %xmm0, (%rdi) -; AVX1-NEXT: retq - -; AVX2: @test3 -; AVX2: # BB#0: -; AVX2-NEXT: vmovd %esi, %xmm0 -; AVX2-NEXT: vpbroadcastw %xmm0, %xmm0 -; AVX2-NEXT: vmovdqu (%rdi), %xmm1 -; AVX2-NEXT: vpsubusw %xmm0, %xmm1, %xmm0 -; AVX2-NEXT: vmovdqu %xmm0, (%rdi) -; AVX2-NEXT: retq } define void @test4(i8* nocapture %head) nounwind { +; SSE-LABEL: test4: +; SSE: ## BB#0: ## %vector.ph +; SSE-NEXT: movdqu (%rdi), %xmm0 +; SSE-NEXT: psubusb {{.*}}(%rip), %xmm0 +; SSE-NEXT: movdqu %xmm0, (%rdi) +; SSE-NEXT: retq +; +; AVX-LABEL: test4: +; AVX: ## BB#0: ## %vector.ph +; AVX-NEXT: vmovdqu (%rdi), %xmm0 +; AVX-NEXT: vpsubusb {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vmovdqu %xmm0, (%rdi) +; AVX-NEXT: retq vector.ph: %0 = getelementptr inbounds i8, i8* %head, i64 0 %1 = bitcast i8* %0 to <16 x i8>* @@ -122,30 +127,22 @@ vector.ph: %5 = select <16 x i1> %3, <16 x i8> %4, <16 x i8> zeroinitializer store <16 x i8> %5, <16 x i8>* %1, align 1 ret void - -; SSSE3: @test4 -; SSSE3: # BB#0: -; SSSE3-NEXT: movdqu (%rdi), %xmm0 -; SSSE3-NEXT: psubusb LCPI3_0(%rip), %xmm0 -; SSSE3-NEXT: movdqu %xmm0, (%rdi) -; SSSE3-NEXT: retq - -; AVX1: @test4 -; AVX1: # BB#0: -; AVX1-NEXT: vmovdqu (%rdi), %xmm0 -; AVX1-NEXT: vpsubusb LCPI3_0(%rip), %xmm0, %xmm0 -; AVX1-NEXT: vmovdqu %xmm0, (%rdi) -; AVX1-NEXT: retq - -; AVX2: @test4 -; AVX2: # BB#0: -; AVX2-NEXT: vmovdqu (%rdi), %xmm0 -; AVX2-NEXT: vpsubusb LCPI3_0(%rip), %xmm0, %xmm0 -; AVX2-NEXT: vmovdqu %xmm0, (%rdi) -; AVX2-NEXT: retq } define void @test5(i8* nocapture %head) nounwind { +; SSE-LABEL: test5: +; SSE: ## BB#0: ## %vector.ph +; SSE-NEXT: movdqu (%rdi), %xmm0 +; SSE-NEXT: psubusb {{.*}}(%rip), %xmm0 +; SSE-NEXT: movdqu %xmm0, (%rdi) +; SSE-NEXT: retq +; +; AVX-LABEL: test5: +; AVX: ## BB#0: ## %vector.ph +; AVX-NEXT: vmovdqu (%rdi), %xmm0 +; AVX-NEXT: vpsubusb {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vmovdqu %xmm0, (%rdi) +; AVX-NEXT: retq vector.ph: %0 = getelementptr inbounds i8, i8* %head, i64 0 %1 = bitcast i8* %0 to <16 x i8>* @@ -155,30 +152,49 @@ vector.ph: %5 = select <16 x i1> %3, <16 x i8> %4, <16 x i8> zeroinitializer store <16 x i8> %5, <16 x i8>* %1, align 1 ret void - -; SSSE3: @test5 -; SSSE3: # BB#0: -; SSSE3-NEXT: movdqu (%rdi), %xmm0 -; SSSE3-NEXT: psubusb LCPI4_0(%rip), %xmm0 -; SSSE3-NEXT: movdqu %xmm0, (%rdi) -; SSSE3-NEXT: retq - -; AVX1: @test5 -; AVX1: # BB#0: -; AVX1-NEXT: vmovdqu (%rdi), %xmm0 -; AVX1-NEXT: vpsubusb LCPI4_0(%rip), %xmm0 -; AVX1-NEXT: vmovdqu %xmm0, (%rdi) -; AVX1-NEXT: retq - -; AVX2: @test5 -; AVX2: # BB#0: -; AVX2-NEXT: vmovdqu (%rdi), %xmm0 -; AVX2-NEXT: vpsubusb LCPI4_0(%rip), %xmm0 -; AVX2-NEXT: vmovdqu %xmm0, (%rdi) -; AVX2-NEXT: retq } define void @test6(i8* nocapture %head, i8 zeroext %w) nounwind { +; SSE2-LABEL: test6: +; SSE2: ## BB#0: ## %vector.ph +; SSE2-NEXT: movd %esi, %xmm0 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] +; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4] +; SSE2-NEXT: movdqu (%rdi), %xmm1 +; SSE2-NEXT: psubusb %xmm0, %xmm1 +; SSE2-NEXT: movdqu %xmm1, (%rdi) +; SSE2-NEXT: retq +; +; SSSE3-LABEL: test6: +; SSSE3: ## BB#0: ## %vector.ph +; SSSE3-NEXT: movd %esi, %xmm0 +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: pshufb %xmm1, %xmm0 +; SSSE3-NEXT: movdqu (%rdi), %xmm1 +; SSSE3-NEXT: psubusb %xmm0, %xmm1 +; SSSE3-NEXT: movdqu %xmm1, (%rdi) +; SSSE3-NEXT: retq +; +; AVX1-LABEL: test6: +; AVX1: ## BB#0: ## %vector.ph +; AVX1-NEXT: vmovd %esi, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vmovdqu (%rdi), %xmm1 +; AVX1-NEXT: vpsubusb %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vmovdqu %xmm0, (%rdi) +; AVX1-NEXT: retq +; +; AVX2-LABEL: test6: +; AVX2: ## BB#0: ## %vector.ph +; AVX2-NEXT: vmovd %esi, %xmm0 +; AVX2-NEXT: vpbroadcastb %xmm0, %xmm0 +; AVX2-NEXT: vmovdqu (%rdi), %xmm1 +; AVX2-NEXT: vpsubusb %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vmovdqu %xmm0, (%rdi) +; AVX2-NEXT: retq vector.ph: %0 = insertelement <16 x i8> undef, i8 %w, i32 0 %broadcast15 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> zeroinitializer @@ -190,38 +206,41 @@ vector.ph: %6 = select <16 x i1> %4, <16 x i8> zeroinitializer, <16 x i8> %5 store <16 x i8> %6, <16 x i8>* %2, align 1 ret void - -; SSSE3: @test6 -; SSSE3: # BB#0: -; SSSE3-NEXT: movd %esi, %xmm0 -; SSSE3-NEXT: pxor %xmm1, %xmm1 -; SSSE3-NEXT: pshufb %xmm1, %xmm0 -; SSSE3-NEXT: movdqu (%rdi), %xmm1 -; SSSE3-NEXT: psubusb %xmm0, %xmm1 -; SSSE3-NEXT: movdqu %xmm1, (%rdi) -; SSSE3-NEXT: retq - -; AVX1: @test6 -; AVX1: # BB#0: -; AVX1-NEXT: vmovd %esi, %xmm0 -; AVX1-NEXT: vpxor %xmm1, %xmm1 -; AVX1-NEXT: vpshufb %xmm1, %xmm0 -; AVX1-NEXT: vmovdqu (%rdi), %xmm1 -; AVX1-NEXT: vpsubusb %xmm0, %xmm1, %xmm0 -; AVX1-NEXT: vmovdqu %xmm0, (%rdi) -; AVX1-NEXT: retq - -; AVX2: @test6 -; AVX2: # BB#0: -; AVX2-NEXT: vmovd %esi, %xmm0 -; AVX2-NEXT: vpbroadcastb %xmm0, %xmm0 -; AVX2-NEXT: vmovdqu (%rdi), %xmm1 -; AVX2-NEXT: vpsubusb %xmm0, %xmm1, %xmm0 -; AVX2-NEXT: vmovdqu %xmm0, (%rdi) -; AVX2-NEXT: retq } define void @test7(i16* nocapture %head) nounwind { +; SSE-LABEL: test7: +; SSE: ## BB#0: ## %vector.ph +; SSE-NEXT: movdqu (%rdi), %xmm0 +; SSE-NEXT: movdqu 16(%rdi), %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE-NEXT: psubusw %xmm2, %xmm0 +; SSE-NEXT: psubusw %xmm2, %xmm1 +; SSE-NEXT: movdqu %xmm1, 16(%rdi) +; SSE-NEXT: movdqu %xmm0, (%rdi) +; SSE-NEXT: retq +; +; AVX1-LABEL: test7: +; AVX1: ## BB#0: ## %vector.ph +; AVX1-NEXT: vmovups (%rdi), %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpgtw %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vpcmpgtw %xmm0, %xmm2, %xmm2 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 +; AVX1-NEXT: vxorps {{.*}}(%rip), %ymm0, %ymm0 +; AVX1-NEXT: vandps %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: vmovups %ymm0, (%rdi) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: test7: +; AVX2: ## BB#0: ## %vector.ph +; AVX2-NEXT: vmovdqu (%rdi), %ymm0 +; AVX2-NEXT: vpsubusw {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vmovdqu %ymm0, (%rdi) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq vector.ph: %0 = getelementptr inbounds i16, i16* %head, i64 0 %1 = bitcast i16* %0 to <16 x i16>* @@ -231,17 +250,47 @@ vector.ph: %5 = select <16 x i1> %3, <16 x i16> %4, <16 x i16> zeroinitializer store <16 x i16> %5, <16 x i16>* %1, align 2 ret void - -; AVX2: @test7 -; AVX2: # BB#0: -; AVX2-NEXT: vmovdqu (%rdi), %ymm0 -; AVX2-NEXT: vpsubusw LCPI6_0(%rip), %ymm0, %ymm0 -; AVX2-NEXT: vmovdqu %ymm0, (%rdi) -; AVX2-NEXT: vzeroupper -; AVX2-NEXT: retq } define void @test8(i16* nocapture %head) nounwind { +; SSE-LABEL: test8: +; SSE: ## BB#0: ## %vector.ph +; SSE-NEXT: movdqu (%rdi), %xmm0 +; SSE-NEXT: movdqu 16(%rdi), %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [32767,32767,32767,32767,32767,32767,32767,32767] +; SSE-NEXT: psubusw %xmm2, %xmm0 +; SSE-NEXT: psubusw %xmm2, %xmm1 +; SSE-NEXT: movdqu %xmm1, 16(%rdi) +; SSE-NEXT: movdqu %xmm0, (%rdi) +; SSE-NEXT: retq +; +; AVX1-LABEL: test8: +; AVX1: ## BB#0: ## %vector.ph +; AVX1-NEXT: vmovups (%rdi), %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vmovaps {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768] +; AVX1-NEXT: vxorps %xmm2, %xmm1, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [65534,65534,65534,65534,65534,65534,65534,65534] +; AVX1-NEXT: vpcmpgtw %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vxorps %xmm2, %xmm0, %xmm2 +; AVX1-NEXT: vpcmpgtw %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [32769,32769,32769,32769,32769,32769,32769,32769] +; AVX1-NEXT: vpaddw %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpaddw %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vandps %ymm0, %ymm2, %ymm0 +; AVX1-NEXT: vmovups %ymm0, (%rdi) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: test8: +; AVX2: ## BB#0: ## %vector.ph +; AVX2-NEXT: vmovdqu (%rdi), %ymm0 +; AVX2-NEXT: vpsubusw {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vmovdqu %ymm0, (%rdi) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq vector.ph: %0 = getelementptr inbounds i16, i16* %head, i64 0 %1 = bitcast i16* %0 to <16 x i16>* @@ -252,16 +301,63 @@ vector.ph: store <16 x i16> %5, <16 x i16>* %1, align 2 ret void -; AVX2: @test8 -; AVX2: # BB#0: -; AVX2-NEXT: vmovdqu (%rdi), %ymm0 -; AVX2-NEXT: vpsubusw LCPI7_0(%rip), %ymm0, %ymm0 -; AVX2-NEXT: vmovdqu %ymm0, (%rdi) -; AVX2-NEXT: vzeroupper -; AVX2-NEXT: retq } define void @test9(i16* nocapture %head, i16 zeroext %w) nounwind { +; SSE2-LABEL: test9: +; SSE2: ## BB#0: ## %vector.ph +; SSE2-NEXT: movd %esi, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] +; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4] +; SSE2-NEXT: movdqu (%rdi), %xmm1 +; SSE2-NEXT: movdqu 16(%rdi), %xmm2 +; SSE2-NEXT: psubusw %xmm0, %xmm1 +; SSE2-NEXT: psubusw %xmm0, %xmm2 +; SSE2-NEXT: movdqu %xmm2, 16(%rdi) +; SSE2-NEXT: movdqu %xmm1, (%rdi) +; SSE2-NEXT: retq +; +; SSSE3-LABEL: test9: +; SSSE3: ## BB#0: ## %vector.ph +; SSSE3-NEXT: movd %esi, %xmm0 +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1] +; SSSE3-NEXT: movdqu (%rdi), %xmm1 +; SSSE3-NEXT: movdqu 16(%rdi), %xmm2 +; SSSE3-NEXT: psubusw %xmm0, %xmm1 +; SSSE3-NEXT: psubusw %xmm0, %xmm2 +; SSSE3-NEXT: movdqu %xmm2, 16(%rdi) +; SSSE3-NEXT: movdqu %xmm1, (%rdi) +; SSSE3-NEXT: retq +; +; AVX1-LABEL: test9: +; AVX1: ## BB#0: ## %vector.ph +; AVX1-NEXT: vmovups (%rdi), %ymm0 +; AVX1-NEXT: vmovd %esi, %xmm1 +; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1] +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpsubw %xmm1, %xmm2, %xmm3 +; AVX1-NEXT: vpsubw %xmm1, %xmm0, %xmm4 +; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm3 +; AVX1-NEXT: vpmaxuw %xmm1, %xmm2, %xmm4 +; AVX1-NEXT: vpcmpeqw %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vpmaxuw %xmm1, %xmm0, %xmm1 +; AVX1-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: vandps %ymm3, %ymm0, %ymm0 +; AVX1-NEXT: vmovups %ymm0, (%rdi) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: test9: +; AVX2: ## BB#0: ## %vector.ph +; AVX2-NEXT: vmovd %esi, %xmm0 +; AVX2-NEXT: vpbroadcastw %xmm0, %ymm0 +; AVX2-NEXT: vmovdqu (%rdi), %ymm1 +; AVX2-NEXT: vpsubusw %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: vmovdqu %ymm0, (%rdi) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq vector.ph: %0 = insertelement <16 x i16> undef, i16 %w, i32 0 %broadcast15 = shufflevector <16 x i16> %0, <16 x i16> undef, <16 x i32> zeroinitializer @@ -273,19 +369,41 @@ vector.ph: %6 = select <16 x i1> %4, <16 x i16> zeroinitializer, <16 x i16> %5 store <16 x i16> %6, <16 x i16>* %2, align 2 ret void - -; AVX2: @test9 -; AVX2: # BB#0: -; AVX2-NEXT: vmovd %esi, %xmm0 -; AVX2-NEXT: vpbroadcastw %xmm0, %ymm0 -; AVX2-NEXT: vmovdqu (%rdi), %ymm1 -; AVX2-NEXT: vpsubusw %ymm0, %ymm1, %ymm0 -; AVX2-NEXT: vmovdqu %ymm0, (%rdi) -; AVX2-NEXT: vzeroupper -; AVX2-NEXT: retq } define void @test10(i8* nocapture %head) nounwind { +; SSE-LABEL: test10: +; SSE: ## BB#0: ## %vector.ph +; SSE-NEXT: movdqu (%rdi), %xmm0 +; SSE-NEXT: movdqu 16(%rdi), %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128] +; SSE-NEXT: psubusb %xmm2, %xmm0 +; SSE-NEXT: psubusb %xmm2, %xmm1 +; SSE-NEXT: movdqu %xmm1, 16(%rdi) +; SSE-NEXT: movdqu %xmm0, (%rdi) +; SSE-NEXT: retq +; +; AVX1-LABEL: test10: +; AVX1: ## BB#0: ## %vector.ph +; AVX1-NEXT: vmovups (%rdi), %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpgtb %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vpcmpgtb %xmm0, %xmm2, %xmm2 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 +; AVX1-NEXT: vxorps {{.*}}(%rip), %ymm0, %ymm0 +; AVX1-NEXT: vandps %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: vmovups %ymm0, (%rdi) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: test10: +; AVX2: ## BB#0: ## %vector.ph +; AVX2-NEXT: vmovdqu (%rdi), %ymm0 +; AVX2-NEXT: vpsubusb {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vmovdqu %ymm0, (%rdi) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq vector.ph: %0 = getelementptr inbounds i8, i8* %head, i64 0 %1 = bitcast i8* %0 to <32 x i8>* @@ -296,16 +414,47 @@ vector.ph: store <32 x i8> %5, <32 x i8>* %1, align 1 ret void -; AVX2: @test10 -; AVX2: # BB#0: -; AVX2-NEXT: vmovdqu (%rdi), %ymm0 -; AVX2-NEXT: vpsubusb LCPI9_0(%rip), %ymm0, %ymm0 -; AVX2-NEXT: vmovdqu %ymm0, (%rdi) -; AVX2-NEXT: vzeroupper -; AVX2-NEXT: retq } define void @test11(i8* nocapture %head) nounwind { +; SSE-LABEL: test11: +; SSE: ## BB#0: ## %vector.ph +; SSE-NEXT: movdqu (%rdi), %xmm0 +; SSE-NEXT: movdqu 16(%rdi), %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127] +; SSE-NEXT: psubusb %xmm2, %xmm0 +; SSE-NEXT: psubusb %xmm2, %xmm1 +; SSE-NEXT: movdqu %xmm1, 16(%rdi) +; SSE-NEXT: movdqu %xmm0, (%rdi) +; SSE-NEXT: retq +; +; AVX1-LABEL: test11: +; AVX1: ## BB#0: ## %vector.ph +; AVX1-NEXT: vmovups (%rdi), %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vmovaps {{.*#+}} xmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128] +; AVX1-NEXT: vxorps %xmm2, %xmm1, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [254,254,254,254,254,254,254,254,254,254,254,254,254,254,254,254] +; AVX1-NEXT: vpcmpgtb %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vxorps %xmm2, %xmm0, %xmm2 +; AVX1-NEXT: vpcmpgtb %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [129,129,129,129,129,129,129,129,129,129,129,129,129,129,129,129] +; AVX1-NEXT: vpaddb %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpaddb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vandps %ymm0, %ymm2, %ymm0 +; AVX1-NEXT: vmovups %ymm0, (%rdi) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: test11: +; AVX2: ## BB#0: ## %vector.ph +; AVX2-NEXT: vmovdqu (%rdi), %ymm0 +; AVX2-NEXT: vpsubusb {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vmovdqu %ymm0, (%rdi) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq vector.ph: %0 = getelementptr inbounds i8, i8* %head, i64 0 %1 = bitcast i8* %0 to <32 x i8>* @@ -315,17 +464,66 @@ vector.ph: %5 = select <32 x i1> %3, <32 x i8> %4, <32 x i8> zeroinitializer store <32 x i8> %5, <32 x i8>* %1, align 1 ret void - -; AVX2: @test11 -; AVX2: # BB#0: -; AVX2-NEXT: vmovdqu (%rdi), %ymm0 -; AVX2-NEXT: vpsubusb LCPI10_0(%rip), %ymm0, %ymm0 -; AVX2-NEXT: vmovdqu %ymm0, (%rdi) -; AVX2-NEXT: vzeroupper -; AVX2-NEXT: retq } define void @test12(i8* nocapture %head, i8 zeroext %w) nounwind { +; SSE2-LABEL: test12: +; SSE2: ## BB#0: ## %vector.ph +; SSE2-NEXT: movd %esi, %xmm0 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] +; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4] +; SSE2-NEXT: movdqu (%rdi), %xmm1 +; SSE2-NEXT: movdqu 16(%rdi), %xmm2 +; SSE2-NEXT: psubusb %xmm0, %xmm1 +; SSE2-NEXT: psubusb %xmm0, %xmm2 +; SSE2-NEXT: movdqu %xmm2, 16(%rdi) +; SSE2-NEXT: movdqu %xmm1, (%rdi) +; SSE2-NEXT: retq +; +; SSSE3-LABEL: test12: +; SSSE3: ## BB#0: ## %vector.ph +; SSSE3-NEXT: movd %esi, %xmm0 +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: pshufb %xmm1, %xmm0 +; SSSE3-NEXT: movdqu (%rdi), %xmm1 +; SSSE3-NEXT: movdqu 16(%rdi), %xmm2 +; SSSE3-NEXT: psubusb %xmm0, %xmm1 +; SSSE3-NEXT: psubusb %xmm0, %xmm2 +; SSSE3-NEXT: movdqu %xmm2, 16(%rdi) +; SSSE3-NEXT: movdqu %xmm1, (%rdi) +; SSSE3-NEXT: retq +; +; AVX1-LABEL: test12: +; AVX1: ## BB#0: ## %vector.ph +; AVX1-NEXT: vmovups (%rdi), %ymm0 +; AVX1-NEXT: vmovd %esi, %xmm1 +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpsubb %xmm1, %xmm2, %xmm3 +; AVX1-NEXT: vpsubb %xmm1, %xmm0, %xmm4 +; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm3 +; AVX1-NEXT: vpmaxub %xmm1, %xmm2, %xmm4 +; AVX1-NEXT: vpcmpeqb %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vpmaxub %xmm1, %xmm0, %xmm1 +; AVX1-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: vandps %ymm3, %ymm0, %ymm0 +; AVX1-NEXT: vmovups %ymm0, (%rdi) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: test12: +; AVX2: ## BB#0: ## %vector.ph +; AVX2-NEXT: vmovd %esi, %xmm0 +; AVX2-NEXT: vpbroadcastb %xmm0, %ymm0 +; AVX2-NEXT: vmovdqu (%rdi), %ymm1 +; AVX2-NEXT: vpsubusb %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: vmovdqu %ymm0, (%rdi) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq vector.ph: %0 = insertelement <32 x i8> undef, i8 %w, i32 0 %broadcast15 = shufflevector <32 x i8> %0, <32 x i8> undef, <32 x i32> zeroinitializer @@ -337,14 +535,4 @@ vector.ph: %6 = select <32 x i1> %4, <32 x i8> zeroinitializer, <32 x i8> %5 store <32 x i8> %6, <32 x i8>* %2, align 1 ret void - -; AVX2: @test12 -; AVX2: # BB#0: -; AVX2-NEXT: vmovd %esi, %xmm0 -; AVX2-NEXT: vpbroadcastb %xmm0, %ymm0 -; AVX2-NEXT: vmovdqu (%rdi), %ymm1 -; AVX2-NEXT: vpsubusb %ymm0, %ymm1, %ymm0 -; AVX2-NEXT: vmovdqu %ymm0, (%rdi) -; AVX2-NEXT: vzeroupper -; AVX2-NEXT: retq } diff --git a/test/CodeGen/X86/push-cfi-debug.ll b/test/CodeGen/X86/push-cfi-debug.ll new file mode 100644 index 000000000000..cc00fab525ab --- /dev/null +++ b/test/CodeGen/X86/push-cfi-debug.ll @@ -0,0 +1,53 @@ +; RUN: llc < %s -mtriple=i686-pc-linux | FileCheck %s + + +; Function Attrs: optsize +declare void @foo(i32, i32) #0 +declare x86_stdcallcc void @stdfoo(i32, i32) #0 + +; CHECK-LABEL: test1: +; CHECK: subl $8, %esp +; CHECK: .cfi_adjust_cfa_offset 8 +; CHECK: pushl $2 +; CHECK: .cfi_adjust_cfa_offset 4 +; CHECK: pushl $1 +; CHECK: .cfi_adjust_cfa_offset 4 +; CHECK: calll foo +; CHECK: addl $16, %esp +; CHECK: .cfi_adjust_cfa_offset -16 +; CHECK: subl $8, %esp +; CHECK: .cfi_adjust_cfa_offset 8 +; CHECK: pushl $4 +; CHECK: .cfi_adjust_cfa_offset 4 +; CHECK: pushl $3 +; CHECK: .cfi_adjust_cfa_offset 4 +; CHECK: calll stdfoo +; CHECK: .cfi_adjust_cfa_offset -8 +; CHECK: addl $8, %esp +; CHECK: .cfi_adjust_cfa_offset -8 +define void @test1() #0 !dbg !4 { +entry: + tail call void @foo(i32 1, i32 2) #1, !dbg !10 + tail call x86_stdcallcc void @stdfoo(i32 3, i32 4) #1, !dbg !11 + ret void, !dbg !12 +} + +attributes #0 = { nounwind optsize } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!7, !8} +!llvm.ident = !{!9} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0 (trunk 250289)", isOptimized: true, runtimeVersion: 0, emissionKind: 1, enums: !2, subprograms: !3) +!1 = !DIFile(filename: "foo.c", directory: "foo") +!2 = !{} +!3 = !{!4} +!4 = distinct !DISubprogram(name: "test1", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: true, variables: !2) +!5 = !DISubroutineType(types: !6) +!6 = !{null} +!7 = !{i32 2, !"Dwarf Version", i32 4} +!8 = !{i32 2, !"Debug Info Version", i32 3} +!9 = !{!"clang version 3.8.0 (trunk 250289)"} +!10 = !DILocation(line: 4, column: 3, scope: !4) +!11 = !DILocation(line: 5, column: 3, scope: !4) +!12 = !DILocation(line: 6, column: 1, scope: !4) diff --git a/test/CodeGen/X86/push-cfi-obj.ll b/test/CodeGen/X86/push-cfi-obj.ll new file mode 100644 index 000000000000..33291ec3318a --- /dev/null +++ b/test/CodeGen/X86/push-cfi-obj.ll @@ -0,0 +1,44 @@ +; RUN: llc < %s -mtriple=i686-pc-linux -filetype=obj | llvm-readobj -s -sr -sd | FileCheck %s -check-prefix=LINUX +; RUN: llc < %s -mtriple=i686-darwin-macosx10.7 -filetype=obj | llvm-readobj -sections | FileCheck -check-prefix=DARWIN %s + +; On darwin, check that we manage to generate the compact unwind section +; DARWIN: Name: __compact_unwind +; DARWIN: Segment: __LD + +; LINUX: Name: .eh_frame +; LINUX-NEXT: Type: SHT_PROGBITS (0x1) +; LINUX-NEXT: Flags [ (0x2) +; LINUX-NEXT: SHF_ALLOC (0x2) +; LINUX-NEXT: ] +; LINUX-NEXT: Address: 0x0 +; LINUX-NEXT: Offset: 0x68 +; LINUX-NEXT: Size: 64 +; LINUX-NEXT: Link: 0 +; LINUX-NEXT: Info: 0 +; LINUX-NEXT: AddressAlignment: 4 +; LINUX-NEXT: EntrySize: 0 +; LINUX-NEXT: Relocations [ +; LINUX-NEXT: ] +; LINUX-NEXT: SectionData ( +; LINUX-NEXT: 0000: 1C000000 00000000 017A504C 5200017C |.........zPLR..|| +; LINUX-NEXT: 0010: 08070000 00000000 1B0C0404 88010000 |................| +; LINUX-NEXT: 0020: 1C000000 24000000 00000000 1D000000 |....$...........| +; LINUX-NEXT: 0030: 04000000 00410E08 8502420D 05432E10 |.....A....B..C..| +; LINUX-NEXT: ) + +declare i32 @__gxx_personality_v0(...) +declare void @good(i32 %a, i32 %b, i32 %c, i32 %d) + +define void @test() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +entry: + invoke void @good(i32 1, i32 2, i32 3, i32 4) + to label %continue unwind label %cleanup +continue: + ret void +cleanup: + landingpad { i8*, i32 } + cleanup + ret void +} + +attributes #0 = { optsize "no-frame-pointer-elim"="true" } diff --git a/test/CodeGen/X86/push-cfi.ll b/test/CodeGen/X86/push-cfi.ll new file mode 100644 index 000000000000..6389708f42cc --- /dev/null +++ b/test/CodeGen/X86/push-cfi.ll @@ -0,0 +1,304 @@ +; RUN: llc < %s -mtriple=i686-pc-linux | FileCheck %s -check-prefix=LINUX -check-prefix=CHECK +; RUN: llc < %s -mtriple=i686-apple-darwin | FileCheck %s -check-prefix=DARWIN -check-prefix=CHECK + +declare i32 @__gxx_personality_v0(...) +declare void @good(i32 %a, i32 %b, i32 %c, i32 %d) +declare void @large(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f) +declare void @empty() + +; When we use an invoke, we expect a .cfi_escape GNU_ARGS_SIZE +; with size 16 before the invocation. Without FP, we also expect +; .cfi_adjust_cfa_offset after each push. +; Darwin should not generate pushes in either circumstance. +; CHECK-LABEL: test1_nofp: +; LINUX: .cfi_escape 0x2e, 0x10 +; LINUX-NEXT: pushl $4 +; LINUX-NEXT: Ltmp{{[0-9]+}}: +; LINUX-NEXT: .cfi_adjust_cfa_offset 4 +; LINUX-NEXT: pushl $3 +; LINUX-NEXT: Ltmp{{[0-9]+}}: +; LINUX-NEXT: .cfi_adjust_cfa_offset 4 +; LINUX-NEXT: pushl $2 +; LINUX-NEXT: Ltmp{{[0-9]+}}: +; LINUX-NEXT: .cfi_adjust_cfa_offset 4 +; LINUX-NEXT: pushl $1 +; LINUX-NEXT: Ltmp{{[0-9]+}}: +; LINUX-NEXT: .cfi_adjust_cfa_offset 4 +; LINUX-NEXT: call +; LINUX-NEXT: addl $16, %esp +; LINUX: .cfi_adjust_cfa_offset -16 +; DARWIN-NOT: .cfi_escape +; DARWIN-NOT: pushl +define void @test1_nofp() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +entry: + invoke void @good(i32 1, i32 2, i32 3, i32 4) + to label %continue unwind label %cleanup +continue: + ret void +cleanup: + landingpad { i8*, i32 } + cleanup + ret void +} + +; CHECK-LABEL: test1_fp: +; LINUX: .cfi_escape 0x2e, 0x10 +; LINUX-NEXT: pushl $4 +; LINUX-NEXT: pushl $3 +; LINUX-NEXT: pushl $2 +; LINUX-NEXT: pushl $1 +; LINUX-NEXT: call +; LINUX-NEXT: addl $16, %esp +; DARWIN: pushl %ebp +; DARWIN-NOT: .cfi_escape +; DARWIN-NOT: pushl +define void @test1_fp() #1 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +entry: + invoke void @good(i32 1, i32 2, i32 3, i32 4) + to label %continue unwind label %cleanup +continue: + ret void +cleanup: + landingpad { i8*, i32 } + cleanup + ret void +} + +; If the function has no handlers, we don't need to generate GNU_ARGS_SIZE, +; even if it has an unwind table. Without FP, we still need cfi_adjust_cfa_offset, +; so darwin should not generate pushes. +; CHECK-LABEL: test2_nofp: +; LINUX-NOT: .cfi_escape +; LINUX: pushl $4 +; LINUX-NEXT: Ltmp{{[0-9]+}}: +; LINUX-NEXT: .cfi_adjust_cfa_offset 4 +; LINUX-NEXT: pushl $3 +; LINUX-NEXT: Ltmp{{[0-9]+}}: +; LINUX-NEXT: .cfi_adjust_cfa_offset 4 +; LINUX-NEXT: pushl $2 +; LINUX-NEXT: Ltmp{{[0-9]+}}: +; LINUX-NEXT: .cfi_adjust_cfa_offset 4 +; LINUX-NEXT: pushl $1 +; LINUX-NEXT: Ltmp{{[0-9]+}}: +; LINUX-NEXT: .cfi_adjust_cfa_offset 4 +; LINUX-NEXT: call +; LINUX-NEXT: addl $16, %esp +; LINUX: .cfi_adjust_cfa_offset -16 +; DARWIN-NOT: .cfi_escape +; DARWIN-NOT: pushl +define void @test2_nofp() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +entry: + call void @good(i32 1, i32 2, i32 3, i32 4) + ret void +} + +; CHECK-LABEL: test2_fp: +; CHECK-NOT: .cfi_escape +; CHECK-NOT: .cfi_adjust_cfa_offset +; CHECK: pushl $4 +; CHECK-NEXT: pushl $3 +; CHECK-NEXT: pushl $2 +; CHECK-NEXT: pushl $1 +; CHECK-NEXT: call +; CHECK-NEXT: addl $24, %esp +define void @test2_fp() #1 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +entry: + call void @good(i32 1, i32 2, i32 3, i32 4) + ret void +} + +; If we did not end up using any pushes, no need for GNU_ARGS_SIZE or +; cfi_adjust_cfa_offset. +; CHECK-LABEL: test3_nofp: +; LINUX-NOT: .cfi_escape +; LINUX-NOT: .cfi_adjust_cfa_offset +; LINUX-NOT: pushl +; LINUX: retl +define void @test3_nofp() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +entry: + invoke void @empty() + to label %continue unwind label %cleanup +continue: + ret void +cleanup: + landingpad { i8*, i32 } + cleanup + ret void +} + +; If we did not end up using any pushes, no need for GNU_ARGS_SIZE or +; cfi_adjust_cfa_offset. +; CHECK-LABEL: test3_fp: +; LINUX: pushl %ebp +; LINUX-NOT: .cfi_escape +; LINUX-NOT: .cfi_adjust_cfa_offset +; LINUX-NOT: pushl +; LINUX: retl +define void @test3_fp() #1 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +entry: + invoke void @empty() + to label %continue unwind label %cleanup +continue: + ret void +cleanup: + landingpad { i8*, i32 } + cleanup + ret void +} + +; Different sized stacks need different GNU_ARGS_SIZEs +; CHECK-LABEL: test4: +; LINUX: .cfi_escape 0x2e, 0x10 +; LINUX-NEXT: pushl $4 +; LINUX-NEXT: pushl $3 +; LINUX-NEXT: pushl $2 +; LINUX-NEXT: pushl $1 +; LINUX-NEXT: call +; LINUX-NEXT: addl $16, %esp +; LINUX: .cfi_escape 0x2e, 0x20 +; LINUX: subl $8, %esp +; LINUX-NEXT: pushl $11 +; LINUX-NEXT: pushl $10 +; LINUX-NEXT: pushl $9 +; LINUX-NEXT: pushl $8 +; LINUX-NEXT: pushl $7 +; LINUX-NEXT: pushl $6 +; LINUX-NEXT: calll large +; LINUX-NEXT: addl $32, %esp +define void @test4() #1 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +entry: + invoke void @good(i32 1, i32 2, i32 3, i32 4) + to label %continue1 unwind label %cleanup +continue1: + invoke void @large(i32 6, i32 7, i32 8, i32 9, i32 10, i32 11) + to label %continue2 unwind label %cleanup +continue2: + ret void +cleanup: + landingpad { i8*, i32 } + cleanup + ret void +} + +; If we did use pushes, we need to reset GNU_ARGS_SIZE before a call +; without parameters, but don't need to adjust the cfa offset +; CHECK-LABEL: test5_nofp: +; LINUX: .cfi_escape 0x2e, 0x10 +; LINUX-NEXT: pushl $4 +; LINUX-NEXT: Ltmp{{[0-9]+}}: +; LINUX-NEXT: .cfi_adjust_cfa_offset 4 +; LINUX-NEXT: pushl $3 +; LINUX-NEXT: Ltmp{{[0-9]+}}: +; LINUX-NEXT: .cfi_adjust_cfa_offset 4 +; LINUX-NEXT: pushl $2 +; LINUX-NEXT: Ltmp{{[0-9]+}}: +; LINUX-NEXT: .cfi_adjust_cfa_offset 4 +; LINUX-NEXT: pushl $1 +; LINUX-NEXT: Ltmp{{[0-9]+}}: +; LINUX-NEXT: .cfi_adjust_cfa_offset 4 +; LINUX-NEXT: call +; LINUX-NEXT: addl $16, %esp +; LINUX: .cfi_adjust_cfa_offset -16 +; LINUX-NOT: .cfi_adjust_cfa_offset +; LINUX: .cfi_escape 0x2e, 0x00 +; LINUX-NOT: .cfi_adjust_cfa_offset +; LINUX: call +define void @test5_nofp() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +entry: + invoke void @good(i32 1, i32 2, i32 3, i32 4) + to label %continue1 unwind label %cleanup +continue1: + invoke void @empty() + to label %continue2 unwind label %cleanup +continue2: + ret void +cleanup: + landingpad { i8*, i32 } + cleanup + ret void +} + +; CHECK-LABEL: test5_fp: +; LINUX: .cfi_escape 0x2e, 0x10 +; LINUX-NEXT: pushl $4 +; LINUX-NEXT: pushl $3 +; LINUX-NEXT: pushl $2 +; LINUX-NEXT: pushl $1 +; LINUX-NEXT: call +; LINUX-NEXT: addl $16, %esp +; LINUX: .cfi_escape 0x2e, 0x00 +; LINUX-NOT: .cfi_adjust_cfa_offset +; LINUX: call +define void @test5_fp() #1 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +entry: + invoke void @good(i32 1, i32 2, i32 3, i32 4) + to label %continue1 unwind label %cleanup +continue1: + invoke void @empty() + to label %continue2 unwind label %cleanup +continue2: + ret void +cleanup: + landingpad { i8*, i32 } + cleanup + ret void +} + +; FIXME: This is actually inefficient - we don't need to repeat the .cfi_escape twice. +; CHECK-LABEL: test6: +; LINUX: .cfi_escape 0x2e, 0x10 +; LINUX: call +; LINUX: .cfi_escape 0x2e, 0x10 +; LINUX: call +define void @test6() #1 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +entry: + invoke void @good(i32 1, i32 2, i32 3, i32 4) + to label %continue1 unwind label %cleanup +continue1: + invoke void @good(i32 5, i32 6, i32 7, i32 8) + to label %continue2 unwind label %cleanup +continue2: + ret void +cleanup: + landingpad { i8*, i32 } + cleanup + ret void +} + +; Darwin should generate pushes in the presense of FP and an unwind table, +; but not FP and invoke. +; CHECK-LABEL: test7: +; DARWIN: pushl %ebp +; DARWIN: movl %esp, %ebp +; DARWIN: .cfi_def_cfa_register %ebp +; DARWIN-NOT: .cfi_adjust_cfa_offset +; DARWIN: pushl $4 +; DARWIN-NEXT: pushl $3 +; DARWIN-NEXT: pushl $2 +; DARWIN-NEXT: pushl $1 +; DARWIN-NEXT: call +define void @test7() #1 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +entry: + call void @good(i32 1, i32 2, i32 3, i32 4) + ret void +} + +; CHECK-LABEL: test8: +; DARWIN: pushl %ebp +; DARWIN: movl %esp, %ebp +; DARWIN-NOT: .cfi_adjust_cfa_offset +; DARWIN-NOT: pushl +define void @test8() #1 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +entry: + invoke void @good(i32 1, i32 2, i32 3, i32 4) + to label %continue unwind label %cleanup +continue: + ret void +cleanup: + landingpad { i8*, i32 } + cleanup + ret void +} + +attributes #0 = { optsize } +attributes #1 = { optsize "no-frame-pointer-elim"="true" } diff --git a/test/CodeGen/X86/ragreedy-hoist-spill.ll b/test/CodeGen/X86/ragreedy-hoist-spill.ll index e7dda5349568..46b65bd24fc0 100644 --- a/test/CodeGen/X86/ragreedy-hoist-spill.ll +++ b/test/CodeGen/X86/ragreedy-hoist-spill.ll @@ -1,6 +1,6 @@ ; RUN: llc < %s -mtriple=x86_64-apple-macosx -regalloc=greedy | FileCheck %s -; This testing case is reduced from 254.gap SyFgets funciton. +; This testing case is reduced from 254.gap SyFgets function. ; We make sure a spill is not hoisted to a hotter outer loop. %struct.TMP.1 = type { %struct.TMP.2*, %struct.TMP.2*, [1024 x i8] } diff --git a/test/CodeGen/X86/rem_crash.ll b/test/CodeGen/X86/rem_crash.ll new file mode 100644 index 000000000000..8363b22ab65f --- /dev/null +++ b/test/CodeGen/X86/rem_crash.ll @@ -0,0 +1,257 @@ +; RUN: llc < %s + +define i8 @test_minsize_uu8(i8 %x) minsize optsize { +entry: + %0 = udiv i8 %x, 10 + %1 = urem i8 %x, 10 + %res = add i8 %0, %1 + ret i8 %res +} + +define i8 @test_minsize_ss8(i8 %x) minsize optsize { +entry: + %0 = sdiv i8 %x, 10 + %1 = srem i8 %x, 10 + %res = add i8 %0, %1 + ret i8 %res +} + +define i8 @test_minsize_us8(i8 %x) minsize optsize { +entry: + %0 = udiv i8 %x, 10 + %1 = srem i8 %x, 10 + %res = add i8 %0, %1 + ret i8 %res +} + +define i8 @test_minsize_su8(i8 %x) minsize optsize { +entry: + %0 = sdiv i8 %x, 10 + %1 = urem i8 %x, 10 + %res = add i8 %0, %1 + ret i8 %res +} + +define i16 @test_minsize_uu16(i16 %x) minsize optsize { +entry: + %0 = udiv i16 %x, 10 + %1 = urem i16 %x, 10 + %res = add i16 %0, %1 + ret i16 %res +} + +define i16 @test_minsize_ss16(i16 %x) minsize optsize { +entry: + %0 = sdiv i16 %x, 10 + %1 = srem i16 %x, 10 + %res = add i16 %0, %1 + ret i16 %res +} + +define i16 @test_minsize_us16(i16 %x) minsize optsize { +entry: + %0 = udiv i16 %x, 10 + %1 = srem i16 %x, 10 + %res = add i16 %0, %1 + ret i16 %res +} + +define i16 @test_minsize_su16(i16 %x) minsize optsize { +entry: + %0 = sdiv i16 %x, 10 + %1 = urem i16 %x, 10 + %res = add i16 %0, %1 + ret i16 %res +} + +define i32 @test_minsize_uu32(i32 %x) minsize optsize { +entry: + %0 = udiv i32 %x, 10 + %1 = urem i32 %x, 10 + %res = add i32 %0, %1 + ret i32 %res +} + +define i32 @test_minsize_ss32(i32 %x) minsize optsize { +entry: + %0 = sdiv i32 %x, 10 + %1 = srem i32 %x, 10 + %res = add i32 %0, %1 + ret i32 %res +} + +define i32 @test_minsize_us32(i32 %x) minsize optsize { +entry: + %0 = udiv i32 %x, 10 + %1 = srem i32 %x, 10 + %res = add i32 %0, %1 + ret i32 %res +} + +define i32 @test_minsize_su32(i32 %x) minsize optsize { +entry: + %0 = sdiv i32 %x, 10 + %1 = urem i32 %x, 10 + %res = add i32 %0, %1 + ret i32 %res +} + +define i64 @test_minsize_uu64(i64 %x) minsize optsize { +entry: + %0 = udiv i64 %x, 10 + %1 = urem i64 %x, 10 + %res = add i64 %0, %1 + ret i64 %res +} + +define i64 @test_minsize_ss64(i64 %x) minsize optsize { +entry: + %0 = sdiv i64 %x, 10 + %1 = srem i64 %x, 10 + %res = add i64 %0, %1 + ret i64 %res +} + +define i64 @test_minsize_us64(i64 %x) minsize optsize { +entry: + %0 = udiv i64 %x, 10 + %1 = srem i64 %x, 10 + %res = add i64 %0, %1 + ret i64 %res +} + +define i64 @test_minsize_su64(i64 %x) minsize optsize { +entry: + %0 = sdiv i64 %x, 10 + %1 = urem i64 %x, 10 + %res = add i64 %0, %1 + ret i64 %res +} + +define i8 @test_uu8(i8 %x) optsize { +entry: + %0 = udiv i8 %x, 10 + %1 = urem i8 %x, 10 + %res = add i8 %0, %1 + ret i8 %res +} + +define i8 @test_ss8(i8 %x) optsize { +entry: + %0 = sdiv i8 %x, 10 + %1 = srem i8 %x, 10 + %res = add i8 %0, %1 + ret i8 %res +} + +define i8 @test_us8(i8 %x) optsize { +entry: + %0 = udiv i8 %x, 10 + %1 = srem i8 %x, 10 + %res = add i8 %0, %1 + ret i8 %res +} + +define i8 @test_su8(i8 %x) optsize { +entry: + %0 = sdiv i8 %x, 10 + %1 = urem i8 %x, 10 + %res = add i8 %0, %1 + ret i8 %res +} + +define i16 @test_uu16(i16 %x) optsize { +entry: + %0 = udiv i16 %x, 10 + %1 = urem i16 %x, 10 + %res = add i16 %0, %1 + ret i16 %res +} + +define i16 @test_ss16(i16 %x) optsize { +entry: + %0 = sdiv i16 %x, 10 + %1 = srem i16 %x, 10 + %res = add i16 %0, %1 + ret i16 %res +} + +define i16 @test_us16(i16 %x) optsize { +entry: + %0 = udiv i16 %x, 10 + %1 = srem i16 %x, 10 + %res = add i16 %0, %1 + ret i16 %res +} + +define i16 @test_su16(i16 %x) optsize { +entry: + %0 = sdiv i16 %x, 10 + %1 = urem i16 %x, 10 + %res = add i16 %0, %1 + ret i16 %res +} + +define i32 @test_uu32(i32 %x) optsize { +entry: + %0 = udiv i32 %x, 10 + %1 = urem i32 %x, 10 + %res = add i32 %0, %1 + ret i32 %res +} + +define i32 @test_ss32(i32 %x) optsize { +entry: + %0 = sdiv i32 %x, 10 + %1 = srem i32 %x, 10 + %res = add i32 %0, %1 + ret i32 %res +} + +define i32 @test_us32(i32 %x) optsize { +entry: + %0 = udiv i32 %x, 10 + %1 = srem i32 %x, 10 + %res = add i32 %0, %1 + ret i32 %res +} + +define i32 @test_su32(i32 %x) optsize { +entry: + %0 = sdiv i32 %x, 10 + %1 = urem i32 %x, 10 + %res = add i32 %0, %1 + ret i32 %res +} + +define i64 @test_uu64(i64 %x) optsize { +entry: + %0 = udiv i64 %x, 10 + %1 = urem i64 %x, 10 + %res = add i64 %0, %1 + ret i64 %res +} + +define i64 @test_ss64(i64 %x) optsize { +entry: + %0 = sdiv i64 %x, 10 + %1 = srem i64 %x, 10 + %res = add i64 %0, %1 + ret i64 %res +} + +define i64 @test_us64(i64 %x) optsize { +entry: + %0 = udiv i64 %x, 10 + %1 = srem i64 %x, 10 + %res = add i64 %0, %1 + ret i64 %res +} + +define i64 @test_su64(i64 %x) optsize { +entry: + %0 = sdiv i64 %x, 10 + %1 = urem i64 %x, 10 + %res = add i64 %0, %1 + ret i64 %res +} diff --git a/test/CodeGen/X86/remat-invalid-liveness.ll b/test/CodeGen/X86/remat-invalid-liveness.ll deleted file mode 100644 index c6b43b0dd3e4..000000000000 --- a/test/CodeGen/X86/remat-invalid-liveness.ll +++ /dev/null @@ -1,85 +0,0 @@ -; RUN: llc %s -mcpu=core2 -o - | FileCheck %s -; This test was failing while tracking the liveness in the register scavenger -; during the branching folding pass. The allocation of the subregisters was -; incorrect. -; I.e., the faulty pattern looked like: -; CH = movb 64 -; ECX = movl 3 <- CH was killed here. -; CH = subb CH, ... -; -; This reduced test case triggers the crash before the fix, but does not -; strictly speaking check that the resulting code is correct. -; To check that the code is actually correct we would need to check the -; liveness of the produced code. -; -; Currently, we check that after ECX = movl 3, we do not have subb CH, -; whereas CH could have been redefine in between and that would have been -; totally fine. -; -target datalayout = "e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128" -target triple = "i386-apple-macosx10.9" - -%struct.A = type { %struct.B, %struct.C, %struct.D*, [1 x i8*] } -%struct.B = type { i32, [4 x i8] } -%struct.C = type { i128 } -%struct.D = type { {}*, [0 x i32] } -%union.E = type { i32 } - -; CHECK-LABEL: __XXX1: -; CHECK: movl $3, %ecx -; CHECK-NOT: subb %{{[a-z]+}}, %ch -; Function Attrs: nounwind optsize ssp -define fastcc void @__XXX1(%struct.A* %ht) #0 { -entry: - %const72 = bitcast i128 72 to i128 - %const3 = bitcast i128 3 to i128 - switch i32 undef, label %if.end196 [ - i32 1, label %sw.bb.i - i32 3, label %sw.bb2.i - ] - -sw.bb.i: ; preds = %entry - %call.i.i.i = tail call i32 undef(%struct.A* %ht, i8 zeroext 22, i32 undef, i32 0, %struct.D* undef) - %bf.load.i.i = load i128, i128* undef, align 4 - %bf.lshr.i.i = lshr i128 %bf.load.i.i, %const72 - %shl1.i.i = shl nuw nsw i128 %bf.lshr.i.i, 8 - %shl.i.i = trunc i128 %shl1.i.i to i32 - br i1 undef, label %cond.false10.i.i, label %__XXX2.exit.i.i - -__XXX2.exit.i.i: ; preds = %sw.bb.i - %extract11.i.i.i = lshr i128 %bf.load.i.i, %const3 - %extract.t12.i.i.i = trunc i128 %extract11.i.i.i to i32 - %bf.cast7.i.i.i = and i32 %extract.t12.i.i.i, 3 - %arrayidx.i.i.i = getelementptr inbounds %struct.A, %struct.A* %ht, i32 0, i32 3, i32 %bf.cast7.i.i.i - br label %cond.end12.i.i - -cond.false10.i.i: ; preds = %sw.bb.i - %arrayidx.i6.i.i = getelementptr inbounds %struct.A, %struct.A* %ht, i32 0, i32 3, i32 0 - br label %cond.end12.i.i - -cond.end12.i.i: ; preds = %cond.false10.i.i, %__XXX2.exit.i.i - %.sink.in.i.i = phi i8** [ %arrayidx.i.i.i, %__XXX2.exit.i.i ], [ %arrayidx.i6.i.i, %cond.false10.i.i ] - %.sink.i.i = load i8*, i8** %.sink.in.i.i, align 4 - %tmp = bitcast i8* %.sink.i.i to %union.E* - br i1 undef, label %for.body.i.i, label %if.end196 - -for.body.i.i: ; preds = %for.body.i.i, %cond.end12.i.i - %weak.i.i = getelementptr inbounds %union.E, %union.E* %tmp, i32 undef, i32 0 - %tmp1 = load i32, i32* %weak.i.i, align 4 - %cmp36.i.i = icmp ne i32 %tmp1, %shl.i.i - %or.cond = and i1 %cmp36.i.i, false - br i1 %or.cond, label %for.body.i.i, label %if.end196 - -sw.bb2.i: ; preds = %entry - %bf.lshr.i85.i = lshr i128 undef, %const72 - br i1 undef, label %if.end196, label %__XXX2.exit.i95.i - -__XXX2.exit.i95.i: ; preds = %sw.bb2.i - %extract11.i.i91.i = lshr i128 undef, %const3 - br label %if.end196 - -if.end196: ; preds = %__XXX2.exit.i95.i, %sw.bb2.i, %for.body.i.i, %cond.end12.i.i, %entry - ret void -} - -attributes #0 = { nounwind optsize ssp "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" } diff --git a/test/CodeGen/X86/rodata-relocs.ll b/test/CodeGen/X86/rodata-relocs.ll index 9228ea1f621f..6379ef1bf737 100644 --- a/test/CodeGen/X86/rodata-relocs.ll +++ b/test/CodeGen/X86/rodata-relocs.ll @@ -32,15 +32,15 @@ target triple = "x86_64-unknown-linux-gnu" ; PIC: .section .rodata.cst16,"aM",@progbits,16 ; PIC: e: ; PIC: e1: -; PIC: .section .data.rel.ro.local,"aw",@progbits +; PIC: .section .data.rel.ro,"aw",@progbits ; PIC: p: ; PIC: t: -; PIC: .section .data.rel.ro,"aw",@progbits +; PIC-NOT: .section ; PIC: p1: ; PIC: t1: -; PIC: .section .data.rel,"aw",@progbits +; PIC: .data ; PIC: p2: ; PIC: t2: -; PIC: .section .data.rel.local,"aw",@progbits +; PIC-NOT: .section ; PIC: p3: ; PIC: t3: diff --git a/test/CodeGen/X86/rounding-ops.ll b/test/CodeGen/X86/rounding-ops.ll index 69f4bfb9f47d..15a11d1d6a96 100644 --- a/test/CodeGen/X86/rounding-ops.ll +++ b/test/CodeGen/X86/rounding-ops.ll @@ -6,10 +6,10 @@ define float @test1(float %x) nounwind { ret float %call ; CHECK-SSE-LABEL: test1: -; CHECK-SSE: roundss $1 +; CHECK-SSE: roundss $9 ; CHECK-AVX-LABEL: test1: -; CHECK-AVX: vroundss $1 +; CHECK-AVX: vroundss $9 } declare float @floorf(float) nounwind readnone @@ -19,10 +19,10 @@ define double @test2(double %x) nounwind { ret double %call ; CHECK-SSE-LABEL: test2: -; CHECK-SSE: roundsd $1 +; CHECK-SSE: roundsd $9 ; CHECK-AVX-LABEL: test2: -; CHECK-AVX: vroundsd $1 +; CHECK-AVX: vroundsd $9 } declare double @floor(double) nounwind readnone @@ -58,10 +58,10 @@ define float @test5(float %x) nounwind { ret float %call ; CHECK-SSE-LABEL: test5: -; CHECK-SSE: roundss $2 +; CHECK-SSE: roundss $10 ; CHECK-AVX-LABEL: test5: -; CHECK-AVX: vroundss $2 +; CHECK-AVX: vroundss $10 } declare float @ceilf(float) nounwind readnone @@ -71,10 +71,10 @@ define double @test6(double %x) nounwind { ret double %call ; CHECK-SSE-LABEL: test6: -; CHECK-SSE: roundsd $2 +; CHECK-SSE: roundsd $10 ; CHECK-AVX-LABEL: test6: -; CHECK-AVX: vroundsd $2 +; CHECK-AVX: vroundsd $10 } declare double @ceil(double) nounwind readnone @@ -110,10 +110,10 @@ define float @test9(float %x) nounwind { ret float %call ; CHECK-SSE-LABEL: test9: -; CHECK-SSE: roundss $3 +; CHECK-SSE: roundss $11 ; CHECK-AVX-LABEL: test9: -; CHECK-AVX: vroundss $3 +; CHECK-AVX: vroundss $11 } declare float @truncf(float) nounwind readnone @@ -123,10 +123,10 @@ define double @test10(double %x) nounwind { ret double %call ; CHECK-SSE-LABEL: test10: -; CHECK-SSE: roundsd $3 +; CHECK-SSE: roundsd $11 ; CHECK-AVX-LABEL: test10: -; CHECK-AVX: vroundsd $3 +; CHECK-AVX: vroundsd $11 } declare double @trunc(double) nounwind readnone diff --git a/test/CodeGen/X86/safestack.ll b/test/CodeGen/X86/safestack.ll new file mode 100644 index 000000000000..1ff9a050aefb --- /dev/null +++ b/test/CodeGen/X86/safestack.ll @@ -0,0 +1,32 @@ +; RUN: llc -mtriple=i386-linux < %s -o - | FileCheck --check-prefix=LINUX-I386 %s +; RUN: llc -mtriple=x86_64-linux < %s -o - | FileCheck --check-prefix=LINUX-X64 %s +; RUN: llc -mtriple=i386-linux-android < %s -o - | FileCheck --check-prefix=ANDROID-I386 %s +; RUN: llc -mtriple=x86_64-linux-android < %s -o - | FileCheck --check-prefix=ANDROID-X64 %s + +define void @_Z1fv() safestack { +entry: + %x = alloca i32, align 4 + %0 = bitcast i32* %x to i8* + call void @_Z7CapturePi(i32* nonnull %x) + ret void +} + +declare void @_Z7CapturePi(i32*) + +; LINUX-X64: movq __safestack_unsafe_stack_ptr@GOTTPOFF(%rip), %[[A:.*]] +; LINUX-X64: movq %fs:(%[[A]]), %[[B:.*]] +; LINUX-X64: leaq -16(%[[B]]), %[[C:.*]] +; LINUX-X64: movq %[[C]], %fs:(%[[A]]) + +; LINUX-I386: movl __safestack_unsafe_stack_ptr@INDNTPOFF, %[[A:.*]] +; LINUX-I386: movl %gs:(%[[A]]), %[[B:.*]] +; LINUX-I386: leal -16(%[[B]]), %[[C:.*]] +; LINUX-I386: movl %[[C]], %gs:(%[[A]]) + +; ANDROID-I386: movl %gs:36, %[[A:.*]] +; ANDROID-I386: leal -16(%[[A]]), %[[B:.*]] +; ANDROID-I386: movl %[[B]], %gs:36 + +; ANDROID-X64: movq %fs:72, %[[A:.*]] +; ANDROID-X64: leaq -16(%[[A]]), %[[B:.*]] +; ANDROID-X64: movq %[[B]], %fs:72 diff --git a/test/CodeGen/X86/sar_fold.ll b/test/CodeGen/X86/sar_fold.ll new file mode 100644 index 000000000000..bd0d0c7057d3 --- /dev/null +++ b/test/CodeGen/X86/sar_fold.ll @@ -0,0 +1,37 @@ +; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s + +define i32 @shl16sar15(i32 %a) #0 { +; CHECK-LABEL: shl16sar15: +; CHECK: # BB#0: +; CHECK-NEXT: movswl {{[0-9]+}}(%esp), %eax + %1 = shl i32 %a, 16 + %2 = ashr exact i32 %1, 15 + ret i32 %2 +} + +define i32 @shl16sar17(i32 %a) #0 { +; CHECK-LABEL: shl16sar17: +; CHECK: # BB#0: +; CHECK-NEXT: movswl {{[0-9]+}}(%esp), %eax + %1 = shl i32 %a, 16 + %2 = ashr exact i32 %1, 17 + ret i32 %2 +} + +define i32 @shl24sar23(i32 %a) #0 { +; CHECK-LABEL: shl24sar23: +; CHECK: # BB#0: +; CHECK-NEXT: movsbl {{[0-9]+}}(%esp), %eax + %1 = shl i32 %a, 24 + %2 = ashr exact i32 %1, 23 + ret i32 %2 +} + +define i32 @shl24sar25(i32 %a) #0 { +; CHECK-LABEL: shl24sar25: +; CHECK: # BB#0: +; CHECK-NEXT: movsbl {{[0-9]+}}(%esp), %eax + %1 = shl i32 %a, 24 + %2 = ashr exact i32 %1, 25 + ret i32 %2 +} diff --git a/test/CodeGen/X86/sar_fold64.ll b/test/CodeGen/X86/sar_fold64.ll new file mode 100644 index 000000000000..7b33bb8c0616 --- /dev/null +++ b/test/CodeGen/X86/sar_fold64.ll @@ -0,0 +1,43 @@ +; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s + +define i32 @shl48sar47(i64 %a) #0 { +; CHECK-LABEL: shl48sar47: +; CHECK: # BB#0: +; CHECK-NEXT: movswq %di, %rax + %1 = shl i64 %a, 48 + %2 = ashr exact i64 %1, 47 + %3 = trunc i64 %2 to i32 + ret i32 %3 +} + +define i32 @shl48sar49(i64 %a) #0 { +; CHECK-LABEL: shl48sar49: +; CHECK: # BB#0: +; CHECK-NEXT: movswq %di, %rax + %1 = shl i64 %a, 48 + %2 = ashr exact i64 %1, 49 + %3 = trunc i64 %2 to i32 + ret i32 %3 +} + +define i32 @shl56sar55(i64 %a) #0 { +; CHECK-LABEL: shl56sar55: +; CHECK: # BB#0: +; CHECK-NEXT: movsbq %dil, %rax + %1 = shl i64 %a, 56 + %2 = ashr exact i64 %1, 55 + %3 = trunc i64 %2 to i32 + ret i32 %3 +} + +define i32 @shl56sar57(i64 %a) #0 { +; CHECK-LABEL: shl56sar57: +; CHECK: # BB#0: +; CHECK-NEXT: movsbq %dil, %rax + %1 = shl i64 %a, 56 + %2 = ashr exact i64 %1, 57 + %3 = trunc i64 %2 to i32 + ret i32 %3 +} + +attributes #0 = { nounwind } diff --git a/test/CodeGen/X86/scalar-fp-to-i64.ll b/test/CodeGen/X86/scalar-fp-to-i64.ll new file mode 100644 index 000000000000..d112d2340bdb --- /dev/null +++ b/test/CodeGen/X86/scalar-fp-to-i64.ll @@ -0,0 +1,151 @@ +; Check that scalar FP conversions to signed and unsigned int64 are using +; reasonable sequences, across platforms and target switches. +; +; The signed case is straight forward, and the tests here basically +; ensure successful compilation (f80 with avx512 was broken at one point). +; +; For the unsigned case there are many possible sequences, so to avoid +; a fragile test we just check for the presence of a few key instructions. +; AVX512 on Intel64 can use vcvtts[ds]2usi directly for float and double. +; Otherwise the sequence will involve an FP subtract (fsub, subss or subsd), +; and a truncating conversion (cvtts[ds]2si, fisttp, or fnstcw+fist). When +; both a subtract and fnstcw are needed, they can occur in either order. +; +; The interesting subtargets are AVX512F (vcvtts[ds]2usi), SSE3 (fisttp), +; SSE2 (cvtts[ds]2si) and vanilla X87 (fnstcw+fist, 32-bit only). +; +; RUN: llc < %s -mtriple=i386-pc-windows-msvc -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512_32 +; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512_32 +; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512_64 +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512_64 +; RUN: llc < %s -mtriple=i386-pc-windows-msvc -mattr=+sse3 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE3_32 +; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=+sse3 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE3_32 +; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc -mattr=+sse3 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE3_64 +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse3 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE3_64 +; RUN: llc < %s -mtriple=i386-pc-windows-msvc -mattr=+sse2 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE2_32 +; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=+sse2 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE2_32 +; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc -mattr=+sse2 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE2_64 +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE2_64 +; RUN: llc < %s -mtriple=i386-pc-windows-msvc -mattr=-sse | FileCheck %s --check-prefix=CHECK --check-prefix=X87 +; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=-sse | FileCheck %s --check-prefix=CHECK --check-prefix=X87 + +; CHECK-LABEL: f_to_u64 +; X87-DAG: fsub +; X87-DAG: fnstcw +; X87: fist +; SSE2_32-DAG: {{subss|fsub}} +; SSE2_32-DAG: fnstcw +; SSE2_32: fist +; SSE2_64: subss +; SSE2_64: cvttss2si +; SSE3_32: {{subss|fsub}} +; SSE3_32: fistt +; SSE3_64: subss +; SSE3_64: cvttss2si +; AVX512_32: {{subss|fsub}} +; AVX512_32: fistt +; AVX512_64: vcvttss2usi +; CHECK: ret +define i64 @f_to_u64(float %a) nounwind { + %r = fptoui float %a to i64 + ret i64 %r +} + +; CHECK-LABEL: f_to_s64 +; X87: fnstcw +; X87: fist +; SSE2_32: fnstcw +; SSE2_32: fist +; SSE2_64: cvttss2si +; SSE3_32: fistt +; SSE3_64: cvttss2si +; AVX512_32: fistt +; AVX512_64: vcvttss2si +; CHECK: ret +define i64 @f_to_s64(float %a) nounwind { + %r = fptosi float %a to i64 + ret i64 %r +} + +; CHECK-LABEL: d_to_u64 +; X87-DAG: fsub +; X87-DAG: fnstcw +; X87: fist +; SSE2_32-DAG: {{subsd|fsub}} +; SSE2_32-DAG: fnstcw +; SSE2_32: fist +; SSE2_64: subsd +; SSE2_64: cvttsd2si +; SSE3_32: {{subsd|fsub}} +; SSE3_32: fistt +; SSE3_64: subsd +; SSE3_64: cvttsd2si +; AVX512_32: {{subsd|fsub}} +; AVX512_32: fistt +; AVX512_64: vcvttsd2usi +; CHECK: ret +define i64 @d_to_u64(double %a) nounwind { + %r = fptoui double %a to i64 + ret i64 %r +} + +; CHECK-LABEL: d_to_s64 +; X87: fnstcw +; X87: fist +; SSE2_32: fnstcw +; SSE2_32: fist +; SSE2_64: cvttsd2si +; SSE3_32: fistt +; SSE3_64: cvttsd2si +; AVX512_32: fistt +; AVX512_64: vcvttsd2si +; CHECK: ret +define i64 @d_to_s64(double %a) nounwind { + %r = fptosi double %a to i64 + ret i64 %r +} + +; CHECK-LABEL: x_to_u64 +; CHECK-DAG: fsub +; X87-DAG: fnstcw +; SSE2_32-DAG: fnstcw +; SSE2_64-DAG: fnstcw +; CHECK: fist +; CHECK: ret +define i64 @x_to_u64(x86_fp80 %a) nounwind { + %r = fptoui x86_fp80 %a to i64 + ret i64 %r +} + +; CHECK-LABEL: x_to_s64 +; X87: fnstcw +; X87: fist +; SSE2_32: fnstcw +; SSE2_32: fist +; SSE2_64: fnstcw +; SSE2_64: fist +; SSE3_32: fistt +; SSE3_64: fistt +; AVX512_32: fistt +; AVX512_64: fistt +; CHECK: ret +define i64 @x_to_s64(x86_fp80 %a) nounwind { + %r = fptosi x86_fp80 %a to i64 + ret i64 %r +} + +; CHECK-LABEL: t_to_u64 +; CHECK: __fixunstfdi +; CHECK: ret +define i64 @t_to_u64(fp128 %a) nounwind { + %r = fptoui fp128 %a to i64 + ret i64 %r +} + +; CHECK-LABEL: t_to_s64 +; CHECK: __fixtfdi +; CHECK: ret +define i64 @t_to_s64(fp128 %a) nounwind { + %r = fptosi fp128 %a to i64 + ret i64 %r +} diff --git a/test/CodeGen/X86/scalar-int-to-fp.ll b/test/CodeGen/X86/scalar-int-to-fp.ll new file mode 100644 index 000000000000..93039859cdfb --- /dev/null +++ b/test/CodeGen/X86/scalar-int-to-fp.ll @@ -0,0 +1,132 @@ +; Verify that scalar integer conversions to FP compile successfully +; (at one time long double failed with avx512f), and that reasonable +; instruction sequences are selected based on subtarget features. +; Due to the plethora of reasonable sequences we just check for +; one key instruction, usually a cvt or fild, allowing the test +; to be relatively easily updated when sequences are improved. +; +; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512_32 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512_64 +; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE2_32 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE2_64 +; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=-sse | FileCheck %s --check-prefix=CHECK --check-prefix=X87 + +; CHECK-LABEL: u32_to_f +; AVX512_32: vcvtusi2ssl +; AVX512_64: vcvtusi2ssl +; SSE2_32: cvtsd2ss +; SSE2_64: cvtsi2ssq +; X87: fildll +define float @u32_to_f(i32 %a) nounwind { + %r = uitofp i32 %a to float + ret float %r +} + +; CHECK-LABEL: s32_to_f +; AVX512_32: vcvtsi2ssl +; AVX512_64: vcvtsi2ssl +; SSE2_32: cvtsi2ssl +; SSE2_64: cvtsi2ssl +; X87: fildl +define float @s32_to_f(i32 %a) nounwind { + %r = sitofp i32 %a to float + ret float %r +} + +; CHECK-LABEL: u32_to_d +; AVX512_32: vcvtusi2sdl +; AVX512_64: vcvtusi2sdl +; SSE2_32: subsd +; SSE2_64: cvtsi2sdq +; X87: fildll +define double @u32_to_d(i32 %a) nounwind { + %r = uitofp i32 %a to double + ret double %r +} + +; CHECK-LABEL: s32_to_d +; AVX512_32: vcvtsi2sdl +; AVX512_64: vcvtsi2sdl +; SSE2_32: cvtsi2sdl +; SSE2_64: cvtsi2sdl +; X87: fildl +define double @s32_to_d(i32 %a) nounwind { + %r = sitofp i32 %a to double + ret double %r +} + +; CHECK-LABEL: u32_to_x +; AVX512_32: vsubsd +; AVX512_64: vsubsd +; SSE2_32: subsd +; SSE2_64: fildll +; X87: fildll +define x86_fp80 @u32_to_x(i32 %a) nounwind { + %r = uitofp i32 %a to x86_fp80 + ret x86_fp80 %r +} + +; CHECK-LABEL: s32_to_x +; CHECK: fildl +define x86_fp80 @s32_to_x(i32 %a) nounwind { + %r = sitofp i32 %a to x86_fp80 + ret x86_fp80 %r +} + +; CHECK-LABEL: u64_to_f +; AVX512_32: fildll +; AVX512_64: vcvtusi2ssq +; SSE2_32: fildll +; SSE2_64: cvtsi2ssq +; X87: fildll +define float @u64_to_f(i64 %a) nounwind { + %r = uitofp i64 %a to float + ret float %r +} + +; CHECK-LABEL: s64_to_f +; AVX512_32: fildll +; AVX512_64: vcvtsi2ssq +; SSE2_32: fildll +; SSE2_64: cvtsi2ssq +; X87: fildll +define float @s64_to_f(i64 %a) nounwind { + %r = sitofp i64 %a to float + ret float %r +} + +; CHECK-LABEL: u64_to_d +; AVX512_32: vpunpckldq +; AVX512_64: vcvtusi2sdq +; SSE2_32: punpckldq +; SSE2_64: punpckldq +; X87: fildll +define double @u64_to_d(i64 %a) nounwind { + %r = uitofp i64 %a to double + ret double %r +} + +; CHECK-LABEL: s64_to_d +; AVX512_32: fildll +; AVX512_64: vcvtsi2sdq +; SSE2_32: fildll +; SSE2_64: cvtsi2sdq +; X87: fildll +define double @s64_to_d(i64 %a) nounwind { + %r = sitofp i64 %a to double + ret double %r +} + +; CHECK-LABEL: u64_to_x +; CHECK: fildll +define x86_fp80 @u64_to_x(i64 %a) nounwind { + %r = uitofp i64 %a to x86_fp80 + ret x86_fp80 %r +} + +; CHECK-LABEL: s64_to_x +; CHECK: fildll +define x86_fp80 @s64_to_x(i64 %a) nounwind { + %r = sitofp i64 %a to x86_fp80 + ret x86_fp80 %r +} diff --git a/test/CodeGen/X86/sdiv-pow2.ll b/test/CodeGen/X86/sdiv-pow2.ll new file mode 100644 index 000000000000..e89f76931e18 --- /dev/null +++ b/test/CodeGen/X86/sdiv-pow2.ll @@ -0,0 +1,33 @@ +; RUN: llc -march=x86 < %s | FileCheck %s + +; No attributes, should not use idiv +define i32 @test1(i32 inreg %x) { +entry: + %div = sdiv i32 %x, 16 + ret i32 %div +; CHECK-LABEL: test1: +; CHECK-NOT: idivl +; CHECK: ret +} + +; Has minsize (-Oz) attribute, should generate idiv +define i32 @test2(i32 inreg %x) minsize { +entry: + %div = sdiv i32 %x, 16 + ret i32 %div +; CHECK-LABEL: test2: +; CHECK: idivl +; CHECK: ret +} + +; Has optsize (-Os) attribute, should not generate idiv +define i32 @test3(i32 inreg %x) optsize { +entry: + %div = sdiv i32 %x, 16 + ret i32 %div +; CHECK-LABEL: test3: +; CHECK-NOT: idivl +; CHECK: ret +} + + diff --git a/test/CodeGen/X86/seh-catch-all-win32.ll b/test/CodeGen/X86/seh-catch-all-win32.ll index a4ea8ab78c79..e8da7ab971b1 100644 --- a/test/CodeGen/X86/seh-catch-all-win32.ll +++ b/test/CodeGen/X86/seh-catch-all-win32.ll @@ -22,23 +22,16 @@ entry: to label %__try.cont unwind label %lpad lpad: ; preds = %entry - %0 = landingpad { i8*, i32 } - catch i8* bitcast (i32 ()* @"filt$main" to i8*) - %1 = extractvalue { i8*, i32 } %0, 1 - %2 = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @"filt$main" to i8*)) #4 - %matches = icmp eq i32 %1, %2 - br i1 %matches, label %__except, label %eh.resume + %cs1 = catchswitch within none [label %__except] unwind to caller __except: ; preds = %lpad - %3 = load i32, i32* %__exceptioncode, align 4 - %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([27 x i8], [27 x i8]* @str, i32 0, i32 0), i32 %3) #4 - br label %__try.cont + %p = catchpad within %cs1 [i8* bitcast (i32 ()* @"filt$main" to i8*)] + %code = load i32, i32* %__exceptioncode, align 4 + %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([27 x i8], [27 x i8]* @str, i32 0, i32 0), i32 %code) #4 [ "funclet"(token %p) ] + catchret from %p to label %__try.cont __try.cont: ; preds = %entry, %__except ret i32 0 - -eh.resume: ; preds = %lpad - resume { i8*, i32 } %0 } define internal i32 @"filt$main"() { @@ -68,33 +61,31 @@ entry: ; CHECK: pushl %esi ; CHECK: Lmain$frame_escape_0 = [[code_offs:[-0-9]+]] -; CHECK: Lmain$frame_escape_1 = [[reg_offs:[-0-9]+]] -; CHECK: movl %esp, [[reg_offs]](%ebp) +; CHECK: movl %esp, [[reg_offs:[-0-9]+]](%ebp) ; CHECK: movl $L__ehtable$main, -; EH state 0 +; EH state 0 ; CHECK: movl $0, -16(%ebp) ; CHECK: calll _crash ; CHECK: popl %esi ; CHECK: popl %edi ; CHECK: popl %ebx ; CHECK: retl -; CHECK: # Block address taken -; stackrestore +; CHECK: LBB0_[[lpbb:[0-9]+]]: # %__except{{$}} +; stackrestore ; CHECK: movl -24(%ebp), %esp -; EH state -1 +; EH state -1 ; CHECK: movl [[code_offs]](%ebp), %[[code:[a-z]+]] -; CHECK: movl $-1, -16(%ebp) ; CHECK-DAG: movl %[[code]], 4(%esp) ; CHECK-DAG: movl $_str, (%esp) ; CHECK: calll _printf ; CHECK: .section .xdata,"dr" -; CHECK: Lmain$parent_frame_offset = Lmain$frame_escape_1 +; CHECK: Lmain$parent_frame_offset = [[reg_offs]] ; CHECK: .align 4 ; CHECK: L__ehtable$main ; CHECK-NEXT: .long -1 ; CHECK-NEXT: .long _filt$main -; CHECK-NEXT: .long Ltmp{{[0-9]+}} +; CHECK-NEXT: .long LBB0_[[lpbb]] ; CHECK-LABEL: _filt$main: ; CHECK: pushl %ebp diff --git a/test/CodeGen/X86/seh-catch-all.ll b/test/CodeGen/X86/seh-catch-all.ll index 1c1a3c2139d6..c6a2e4a1094a 100644 --- a/test/CodeGen/X86/seh-catch-all.ll +++ b/test/CodeGen/X86/seh-catch-all.ll @@ -2,6 +2,7 @@ @str = linkonce_odr unnamed_addr constant [27 x i8] c"GetExceptionCode(): 0x%lx\0A\00", align 1 +declare i32 @llvm.eh.exceptioncode(token) declare i32 @__C_specific_handler(...) declare void @crash() declare i32 @printf(i8* nocapture readonly, ...) nounwind @@ -11,20 +12,17 @@ entry: invoke void @crash() to label %__try.cont unwind label %lpad -lpad: - %0 = landingpad { i8*, i32 } - catch i8* null - %1 = extractvalue { i8*, i32 } %0, 0 - %2 = ptrtoint i8* %1 to i64 - %3 = trunc i64 %2 to i32 - call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([27 x i8], [27 x i8]* @str, i64 0, i64 0), i32 %3) - br label %__try.cont - __try.cont: ret i32 0 -eh.resume: - resume { i8*, i32 } %0 +lpad: + %cs1 = catchswitch within none [label %catchall] unwind to caller + +catchall: + %p = catchpad within %cs1 [i8* null, i32 64, i8* null] + %code = call i32 @llvm.eh.exceptioncode(token %p) + call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([27 x i8], [27 x i8]* @str, i64 0, i64 0), i32 %code) [ "funclet"(token %p) ] + catchret from %p to label %__try.cont } ; Check that we can get the exception code from eax to the printf. @@ -32,14 +30,17 @@ eh.resume: ; CHECK-LABEL: main: ; CHECK: callq crash ; CHECK: retq -; CHECK: # Block address taken +; CHECK: .LBB0_2: # %catchall ; CHECK: leaq str(%rip), %rcx ; CHECK: movl %eax, %edx ; CHECK: callq printf ; CHECK: .seh_handlerdata -; CHECK-NEXT: .long 1 +; CHECK-NEXT: .Lmain$parent_frame_offset +; CHECK-NEXT: .long (.Llsda_end0-.Llsda_begin0)/16 +; CHECK-NEXT: .Llsda_begin0: ; CHECK-NEXT: .long .Ltmp{{[0-9]+}}@IMGREL ; CHECK-NEXT: .long .Ltmp{{[0-9]+}}@IMGREL+1 ; CHECK-NEXT: .long 1 -; CHECK-NEXT: .long .Ltmp{{[0-9]+}}@IMGREL +; CHECK-NEXT: .long .LBB0_2@IMGREL +; CHECK-NEXT: .Llsda_end0: diff --git a/test/CodeGen/X86/seh-catchpad.ll b/test/CodeGen/X86/seh-catchpad.ll new file mode 100644 index 000000000000..d9b4c5c6bcf5 --- /dev/null +++ b/test/CodeGen/X86/seh-catchpad.ll @@ -0,0 +1,198 @@ +; RUN: llc < %s | FileCheck %s + +; Based on the source: +; extern "C" int puts(const char *); +; extern "C" int printf(const char *, ...); +; extern "C" int do_div(int a, int b) { return a / b; } +; extern "C" int filt(); +; int main() { +; __try { +; __try { +; do_div(1, 0); +; } __except (1) { +; __try { +; do_div(1, 0); +; } __finally { +; puts("finally"); +; } +; } +; } __except (filt()) { +; puts("caught"); +; } +; return 0; +; } + +; ModuleID = 't.cpp' +target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc" + +$"\01??_C@_07MKBLAIAL@finally?$AA@" = comdat any + +$"\01??_C@_06IBDBCMGJ@caught?$AA@" = comdat any + +@"\01??_C@_07MKBLAIAL@finally?$AA@" = linkonce_odr unnamed_addr constant [8 x i8] c"finally\00", comdat, align 1 +@"\01??_C@_06IBDBCMGJ@caught?$AA@" = linkonce_odr unnamed_addr constant [7 x i8] c"caught\00", comdat, align 1 + +; Function Attrs: nounwind readnone +define i32 @do_div(i32 %a, i32 %b) #0 { +entry: + %div = sdiv i32 %a, %b + ret i32 %div +} + +define i32 @main() #1 personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) { +entry: + %call = invoke i32 @do_div(i32 1, i32 0) #4 + to label %__try.cont.12 unwind label %catch.dispatch + +__except.2: ; preds = %__except + %call4 = invoke i32 @do_div(i32 1, i32 0) #4 + to label %invoke.cont.3 unwind label %ehcleanup + +invoke.cont.3: ; preds = %__except.2 + invoke fastcc void @"\01?fin$0@0@main@@"() #4 + to label %__try.cont.12 unwind label %catch.dispatch.7 + +__except.9: ; preds = %__except.ret + %call11 = tail call i32 @puts(i8* nonnull getelementptr inbounds ([7 x i8], [7 x i8]* @"\01??_C@_06IBDBCMGJ@caught?$AA@", i64 0, i64 0)) + br label %__try.cont.12 + +__try.cont.12: ; preds = %invoke.cont.3, %entry, %__except.9 + ret i32 0 + +catch.dispatch: ; preds = %entry + %cs1 = catchswitch within none [label %__except] unwind label %catch.dispatch.7 + +__except: ; preds = %catch.dispatch + %cp1 = catchpad within %cs1 [i8* null] + catchret from %cp1 to label %__except.2 + +ehcleanup: ; preds = %__except.2 + %cp2 = cleanuppad within none [] + invoke fastcc void @"\01?fin$0@0@main@@"() #4 [ "funclet"(token %cp2) ] + to label %invoke.cont.6 unwind label %catch.dispatch.7 + +invoke.cont.6: ; preds = %ehcleanup + cleanupret from %cp2 unwind label %catch.dispatch.7 + +catch.dispatch.7: + %cs2 = catchswitch within none [label %__except.ret] unwind to caller + +__except.ret: ; preds = %catch.dispatch.7 + %cp3 = catchpad within %cs2 [i8* bitcast (i32 (i8*, i8*)* @"\01?filt$0@0@main@@" to i8*)] + catchret from %cp3 to label %__except.9 +} + +; CHECK: main: # @main +; CHECK: .seh_proc main +; CHECK: .seh_handler __C_specific_handler, @unwind, @except +; CHECK: pushq %rbp +; CHECK: .seh_pushreg 5 +; CHECK: subq $32, %rsp +; CHECK: .seh_stackalloc 32 +; CHECK: leaq 32(%rsp), %rbp +; CHECK: .seh_setframe 5, 32 +; CHECK: .seh_endprologue +; CHECK: .Ltmp0: +; CHECK: movl $1, %ecx +; CHECK: xorl %edx, %edx +; CHECK: callq do_div +; CHECK: .Ltmp1: +; CHECK: .LBB1_[[epilogue:[0-9]+]]: # %__try.cont.12 +; CHECK: xorl %eax, %eax +; CHECK: addq $32, %rsp +; CHECK: popq %rbp +; CHECK: retq +; CHECK: .LBB1_[[except1bb:[0-9]+]]: # %__except +; CHECK: .Ltmp2: +; CHECK: movl $1, %ecx +; CHECK: xorl %edx, %edx +; CHECK: callq do_div +; CHECK: .Ltmp3: +; CHECK: callq "?fin$0@0@main@@" +; CHECK: jmp .LBB1_[[epilogue]] +; CHECK: .LBB1_[[except2bb:[0-9]+]]: # %__except.ret +; CHECK: leaq "??_C@_06IBDBCMGJ@caught?$AA@"(%rip), %rcx +; CHECK: callq puts +; CHECK: jmp .LBB1_[[epilogue]] + +; CHECK: .seh_handlerdata +; CHECK-NEXT: .Lmain$parent_frame_offset = 32 +; CHECK-NEXT: .long (.Llsda_end0-.Llsda_begin0)/16 +; CHECK-NEXT: .Llsda_begin0: +; CHECK-NEXT: .long .Ltmp0@IMGREL+1 +; CHECK-NEXT: .long .Ltmp1@IMGREL+1 +; CHECK-NEXT: .long 1 +; CHECK-NEXT: .long .LBB1_[[except1bb]]@IMGREL +; CHECK-NEXT: .long .Ltmp0@IMGREL+1 +; CHECK-NEXT: .long .Ltmp1@IMGREL+1 +; CHECK-NEXT: .long "?filt$0@0@main@@"@IMGREL +; CHECK-NEXT: .long .LBB1_[[except2bb]]@IMGREL +; CHECK-NEXT: .long .Ltmp2@IMGREL+1 +; CHECK-NEXT: .long .Ltmp3@IMGREL+1 +; CHECK-NEXT: .long "?dtor$[[finbb:[0-9]+]]@?0?main@4HA"@IMGREL +; CHECK-NEXT: .long 0 +; CHECK-NEXT: .long .Ltmp2@IMGREL+1 +; CHECK-NEXT: .long .Ltmp3@IMGREL+1 +; CHECK-NEXT: .long "?filt$0@0@main@@"@IMGREL +; CHECK-NEXT: .long .LBB1_3@IMGREL +; CHECK-NEXT: .long .Ltmp6@IMGREL+1 +; CHECK-NEXT: .long .Ltmp7@IMGREL+1 +; CHECK-NEXT: .long "?filt$0@0@main@@"@IMGREL +; CHECK-NEXT: .long .LBB1_3@IMGREL +; CHECK-NEXT: .Llsda_end0: + +; CHECK: .text +; CHECK: .seh_endproc + +; CHECK: "?dtor$[[finbb]]@?0?main@4HA": +; CHECK: .seh_proc "?dtor$[[finbb]]@?0?main@4HA" +; CHECK: .seh_handler __C_specific_handler, @unwind, @except +; CHECK: .LBB1_[[finbb]]: # %ehcleanup +; CHECK: movq %rdx, 16(%rsp) +; CHECK: pushq %rbp +; CHECK: .seh_pushreg 5 +; CHECK: subq $32, %rsp +; CHECK: .seh_stackalloc 32 +; CHECK: leaq 32(%rdx), %rbp +; CHECK: .seh_endprologue +; CHECK: callq "?fin$0@0@main@@" +; CHECK: nop +; CHECK: addq $32, %rsp +; CHECK: popq %rbp +; CHECK: retq +; CHECK: .seh_handlerdata +; CHECK: .seh_endproc + +define internal i32 @"\01?filt$0@0@main@@"(i8* nocapture readnone %exception_pointers, i8* nocapture readnone %frame_pointer) #1 { +entry: + %call = tail call i32 @filt() + ret i32 %call +} + +; CHECK: "?filt$0@0@main@@": # @"\01?filt$0@0@main@@" +; CHECK: .seh_proc "?filt$0@0@main@@" +; CHECK: .seh_endprologue +; CHECK: rex64 jmp filt # TAILCALL +; CHECK: .seh_handlerdata + +declare i32 @filt() #1 + +declare i32 @__C_specific_handler(...) + +; Function Attrs: noinline nounwind +define internal fastcc void @"\01?fin$0@0@main@@"() #2 { +entry: + %call = tail call i32 @puts(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @"\01??_C@_07MKBLAIAL@finally?$AA@", i64 0, i64 0)) #5 + ret void +} + +; Function Attrs: nounwind +declare i32 @puts(i8* nocapture readonly) #3 + +attributes #0 = { nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #2 = { noinline nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #3 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #4 = { noinline } +attributes #5 = { nounwind } diff --git a/test/CodeGen/X86/seh-except-finally.ll b/test/CodeGen/X86/seh-except-finally.ll index 0630d001bb76..b29788cd015d 100644 --- a/test/CodeGen/X86/seh-except-finally.ll +++ b/test/CodeGen/X86/seh-except-finally.ll @@ -38,84 +38,63 @@ entry: %exn.slot = alloca i8* %ehselector.slot = alloca i32 invoke void @crash() #5 - to label %invoke.cont unwind label %lpad + to label %invoke.cont unwind label %__finally invoke.cont: ; preds = %entry %0 = call i8* @llvm.localaddress() invoke void @"\01?fin$0@0@use_both@@"(i1 zeroext false, i8* %0) #5 - to label %invoke.cont2 unwind label %lpad1 + to label %invoke.cont2 unwind label %catch.dispatch invoke.cont2: ; preds = %invoke.cont br label %__try.cont -lpad: ; preds = %entry - %1 = landingpad { i8*, i32 } - cleanup - catch i8* bitcast (i32 (i8*, i8*)* @"\01?filt$0@0@use_both@@" to i8*) - %2 = extractvalue { i8*, i32 } %1, 0 - store i8* %2, i8** %exn.slot - %3 = extractvalue { i8*, i32 } %1, 1 - store i32 %3, i32* %ehselector.slot - %4 = call i8* @llvm.localaddress() - invoke void @"\01?fin$0@0@use_both@@"(i1 zeroext true, i8* %4) #5 - to label %invoke.cont3 unwind label %lpad1 - -lpad1: ; preds = %lpad, %invoke.cont - %5 = landingpad { i8*, i32 } - catch i8* bitcast (i32 (i8*, i8*)* @"\01?filt$0@0@use_both@@" to i8*) - %6 = extractvalue { i8*, i32 } %5, 0 - store i8* %6, i8** %exn.slot - %7 = extractvalue { i8*, i32 } %5, 1 - store i32 %7, i32* %ehselector.slot - br label %catch.dispatch +__finally: ; preds = %entry + %cleanuppad = cleanuppad within none [] + %locals = call i8* @llvm.localaddress() + invoke void @"\01?fin$0@0@use_both@@"(i1 zeroext true, i8* %locals) #5 [ "funclet"(token %cleanuppad) ] + to label %invoke.cont3 unwind label %catch.dispatch -invoke.cont3: ; preds = %lpad - br label %catch.dispatch +invoke.cont3: ; preds = %__finally + cleanupret from %cleanuppad unwind label %catch.dispatch catch.dispatch: ; preds = %invoke.cont3, %lpad1 - %sel = load i32, i32* %ehselector.slot - %8 = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 (i8*, i8*)* @"\01?filt$0@0@use_both@@" to i8*)) #6 - %matches = icmp eq i32 %sel, %8 - br i1 %matches, label %__except, label %eh.resume + %cs1 = catchswitch within none [label %__except] unwind to caller __except: ; preds = %catch.dispatch - %call = call i32 @puts(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @"\01??_C@_08MLCMLGHM@__except?$AA@", i32 0, i32 0)) - br label %__try.cont + %catchpad = catchpad within %cs1 [i8* bitcast (i32 (i8*, i8*)* @"\01?filt$0@0@use_both@@" to i8*)] + %call = call i32 @puts(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @"\01??_C@_08MLCMLGHM@__except?$AA@", i32 0, i32 0)) [ "funclet"(token %catchpad) ] + catchret from %catchpad to label %__try.cont __try.cont: ; preds = %__except, %invoke.cont2 ret void - -eh.resume: ; preds = %catch.dispatch - %exn = load i8*, i8** %exn.slot - %sel4 = load i32, i32* %ehselector.slot - %lpad.val = insertvalue { i8*, i32 } undef, i8* %exn, 0 - %lpad.val5 = insertvalue { i8*, i32 } %lpad.val, i32 %sel4, 1 - resume { i8*, i32 } %lpad.val5 } ; CHECK-LABEL: use_both: ; CHECK: .Ltmp0 ; CHECK: callq crash ; CHECK: .Ltmp1 -; CHECK: .Ltmp3 -; CHECK: callq "?fin$0@0@use_both@@" ; CHECK: .Ltmp4 +; CHECK: callq "?fin$0@0@use_both@@" +; CHECK: .Ltmp5 ; CHECK: retq ; ; CHECK: .seh_handlerdata -; CHECK-NEXT: .long 3 -; CHECK-NEXT: .long .Ltmp0@IMGREL +; CHECK-NEXT: .Luse_both$parent_frame_offset +; CHECK-NEXT: .long (.Llsda_end0-.Llsda_begin0)/16 +; CHECK-NEXT: .Llsda_begin0: +; CHECK-NEXT: .long .Ltmp0@IMGREL+1 ; CHECK-NEXT: .long .Ltmp1@IMGREL+1 -; CHECK-NEXT: .long "?fin$0@0@use_both@@"@IMGREL +; CHECK-NEXT: .long "?dtor$2@?0?use_both@4HA"@IMGREL ; CHECK-NEXT: .long 0 -; CHECK-NEXT: .long .Ltmp0@IMGREL +; CHECK-NEXT: .long .Ltmp0@IMGREL+1 ; CHECK-NEXT: .long .Ltmp1@IMGREL+1 ; CHECK-NEXT: .long "?filt$0@0@use_both@@"@IMGREL -; CHECK-NEXT: .long .Ltmp{{[0-9]+}}@IMGREL -; CHECK-NEXT: .long .Ltmp3@IMGREL +; CHECK-NEXT: .long .LBB0_{{[0-9]+}}@IMGREL ; CHECK-NEXT: .long .Ltmp4@IMGREL+1 +; CHECK-NEXT: .long .Ltmp5@IMGREL+1 ; CHECK-NEXT: .long "?filt$0@0@use_both@@"@IMGREL -; CHECK-NEXT: .long .Ltmp{{[0-9]+}}@IMGREL +; CHECK-NEXT: .long .LBB0_{{[0-9]+}}@IMGREL +; CHECK-NEXT: .Llsda_end0: ; Function Attrs: noinline nounwind define internal i32 @"\01?filt$0@0@use_both@@"(i8* %exception_pointers, i8* %frame_pointer) #2 { diff --git a/test/CodeGen/X86/seh-exception-code.ll b/test/CodeGen/X86/seh-exception-code.ll new file mode 100644 index 000000000000..20e1544e0b59 --- /dev/null +++ b/test/CodeGen/X86/seh-exception-code.ll @@ -0,0 +1,38 @@ +; RUN: llc -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -verify-machineinstrs -O0 < %s | FileCheck %s + +target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc" + +declare void @f(i32) +declare i32 @__C_specific_handler(...) +declare i32 @llvm.eh.exceptioncode(token) + +define void @ehcode() personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) { +entry: + invoke void @f(i32 0) + to label %__try.cont unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %cs = catchswitch within none [label %__except] unwind to caller + +__except: ; preds = %catch.dispatch + %pad = catchpad within %cs [i8* null] + catchret from %pad to label %__except.1 + +__except.1: ; preds = %__except + %code = call i32 @llvm.eh.exceptioncode(token %pad) + call void @f(i32 %code) + br label %__try.cont + +__try.cont: ; preds = %entry, %__except.1 + ret void +} + +; CHECK-LABEL: ehcode: +; CHECK: xorl %ecx, %ecx +; CHECK: callq f + +; CHECK: # %__except +; CHECK: movl %eax, %ecx +; CHECK-NEXT: callq f diff --git a/test/CodeGen/X86/seh-filter.ll b/test/CodeGen/X86/seh-filter.ll deleted file mode 100644 index 37ed15841a93..000000000000 --- a/test/CodeGen/X86/seh-filter.ll +++ /dev/null @@ -1,21 +0,0 @@ -; RUN: llc -O0 -mtriple=x86_64-windows-msvc < %s | FileCheck %s - -declare void @g() -define void @f() personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) { - invoke void @g() to label %return unwind label %lpad - -return: - ret void - -lpad: - %ehptrs = landingpad {i8*, i32} - filter [0 x i8*] zeroinitializer - call void @__cxa_call_unexpected(i8* null) - unreachable -} -declare i32 @__C_specific_handler(...) -declare void @__cxa_call_unexpected(i8*) - -; We don't emit entries for filters. -; CHECK: .seh_handlerdata -; CHECK: .long 0 diff --git a/test/CodeGen/X86/seh-finally.ll b/test/CodeGen/X86/seh-finally.ll index 350cd932f481..2ef1c984851c 100644 --- a/test/CodeGen/X86/seh-finally.ll +++ b/test/CodeGen/X86/seh-finally.ll @@ -17,50 +17,42 @@ invoke.cont: ; preds = %entry ret i32 0 lpad: ; preds = %entry - %0 = landingpad { i8*, i32 } - cleanup - %1 = extractvalue { i8*, i32 } %0, 0 - %2 = extractvalue { i8*, i32 } %0, 1 - %call2 = invoke i32 @puts(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @str_recovered, i64 0, i64 0)) - to label %invoke.cont1 unwind label %terminate.lpad - -invoke.cont1: ; preds = %lpad - resume { i8*, i32 } %0 - -terminate.lpad: ; preds = %lpad - %3 = landingpad { i8*, i32 } - catch i8* null - call void @abort() - unreachable + %p = cleanuppad within none [] + %call2 = call i32 @puts(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @str_recovered, i64 0, i64 0)) [ "funclet"(token %p) ] + cleanupret from %p unwind to caller } ; X64-LABEL: main: ; X64: retq ; X64: .seh_handlerdata -; X64-NEXT: .long 1 -; X64-NEXT: .long .Ltmp0@IMGREL -; X64-NEXT: .long .Ltmp1@IMGREL -; X64-NEXT: .long main.cleanup@IMGREL -; X64-NEXT: .long 0 - -; X64-LABEL: main.cleanup: +; X64-NEXT: .Lmain$parent_frame_offset = 32 +; X64-NEXT: .long (.Llsda_end0-.Llsda_begin0)/16 # Number of call sites +; X64-NEXT: .Llsda_begin0: +; X64-NEXT: .long .Ltmp0@IMGREL+1 # LabelStart +; X64-NEXT: .long .Ltmp1@IMGREL+1 # LabelEnd +; X64-NEXT: .long "?dtor$2@?0?main@4HA"@IMGREL # FinallyFunclet +; X64-NEXT: .long 0 # Null +; X64-NEXT: .Llsda_end0: + +; X64-LABEL: "?dtor$2@?0?main@4HA": ; X64: callq puts ; X64: retq ; X86-LABEL: _main: ; X86: retl -; X86: .section .xdata,"dr" -; X86: L__ehtable$main: -; X86-NEXT: .long -1 -; X86-NEXT: .long 0 -; X86-NEXT: .long _main.cleanup - -; X86-LABEL: _main.cleanup: +; X86-LABEL: "?dtor$2@?0?main@4HA": +; X86: LBB0_2: ; X86: calll _puts ; X86: retl +; X86: .section .xdata,"dr" +; X86: L__ehtable$main: +; X86-NEXT: .long -1 # ToState +; X86-NEXT: .long 0 # Null +; X86-NEXT: .long "?dtor$2@?0?main@4HA" # FinallyFunclet + declare i32 @__C_specific_handler(...) declare i32 @puts(i8*) diff --git a/test/CodeGen/X86/seh-safe-div-win32.ll b/test/CodeGen/X86/seh-safe-div-win32.ll index b1bcde2c7ff3..643af3a472fb 100644 --- a/test/CodeGen/X86/seh-safe-div-win32.ll +++ b/test/CodeGen/X86/seh-safe-div-win32.ll @@ -28,35 +28,25 @@ entry: %r = alloca i32, align 4 store i32 42, i32* %r invoke void @try_body(i32* %r, i32* %n, i32* %d) - to label %__try.cont unwind label %lpad - -lpad: - %vals = landingpad { i8*, i32 } - catch i8* bitcast (i32 ()* @safe_div_filt0 to i8*) - catch i8* bitcast (i32 ()* @safe_div_filt1 to i8*) - %ehptr = extractvalue { i8*, i32 } %vals, 0 - %sel = extractvalue { i8*, i32 } %vals, 1 - %filt0_val = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @safe_div_filt0 to i8*)) - %is_filt0 = icmp eq i32 %sel, %filt0_val - br i1 %is_filt0, label %handler0, label %eh.dispatch1 - -eh.dispatch1: - %filt1_val = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @safe_div_filt1 to i8*)) - %is_filt1 = icmp eq i32 %sel, %filt1_val - br i1 %is_filt1, label %handler1, label %eh.resume + to label %__try.cont unwind label %lpad0 + +lpad0: + %cs0 = catchswitch within none [label %handler0] unwind label %lpad1 handler0: - call void @puts(i8* getelementptr ([27 x i8], [27 x i8]* @str1, i32 0, i32 0)) + %p0 = catchpad within %cs0 [i8* bitcast (i32 ()* @safe_div_filt0 to i8*)] + call void @puts(i8* getelementptr ([27 x i8], [27 x i8]* @str1, i32 0, i32 0)) [ "funclet"(token %p0) ] store i32 -1, i32* %r, align 4 - br label %__try.cont + catchret from %p0 to label %__try.cont + +lpad1: + %cs1 = catchswitch within none [label %handler1] unwind to caller handler1: - call void @puts(i8* getelementptr ([29 x i8], [29 x i8]* @str2, i32 0, i32 0)) + %p1 = catchpad within %cs1 [i8* bitcast (i32 ()* @safe_div_filt1 to i8*)] + call void @puts(i8* getelementptr ([29 x i8], [29 x i8]* @str2, i32 0, i32 0)) [ "funclet"(token %p1) ] store i32 -2, i32* %r, align 4 - br label %__try.cont - -eh.resume: - resume { i8*, i32 } %vals + catchret from %p1 to label %__try.cont __try.cont: %safe_ret = load i32, i32* %r, align 4 @@ -75,15 +65,13 @@ __try.cont: ; Landing pad code -; CHECK: [[handler0:Ltmp[0-9]+]]: # Block address taken -; CHECK: # %handler0 +; CHECK: [[handler0:LBB0_[0-9]+]]: # %handler0 ; Restore SP ; CHECK: movl {{.*}}(%ebp), %esp ; CHECK: calll _puts ; CHECK: jmp [[cont_bb]] -; CHECK: [[handler1:Ltmp[0-9]+]]: # Block address taken -; CHECK: # %handler1 +; CHECK: [[handler1:LBB0_[0-9]+]]: # %handler1 ; Restore SP ; CHECK: movl {{.*}}(%ebp), %esp ; CHECK: calll _puts diff --git a/test/CodeGen/X86/seh-safe-div.ll b/test/CodeGen/X86/seh-safe-div.ll index 699e58ee8bae..60918cf07058 100644 --- a/test/CodeGen/X86/seh-safe-div.ll +++ b/test/CodeGen/X86/seh-safe-div.ll @@ -27,35 +27,25 @@ define i32 @safe_div(i32* %n, i32* %d) personality i8* bitcast (i32 (...)* @__C_ entry: %r = alloca i32, align 4 invoke void @try_body(i32* %r, i32* %n, i32* %d) - to label %__try.cont unwind label %lpad - -lpad: - %vals = landingpad { i8*, i32 } - catch i8* bitcast (i32 (i8*, i8*)* @safe_div_filt0 to i8*) - catch i8* bitcast (i32 (i8*, i8*)* @safe_div_filt1 to i8*) - %ehptr = extractvalue { i8*, i32 } %vals, 0 - %sel = extractvalue { i8*, i32 } %vals, 1 - %filt0_val = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 (i8*, i8*)* @safe_div_filt0 to i8*)) - %is_filt0 = icmp eq i32 %sel, %filt0_val - br i1 %is_filt0, label %handler0, label %eh.dispatch1 - -eh.dispatch1: - %filt1_val = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 (i8*, i8*)* @safe_div_filt1 to i8*)) - %is_filt1 = icmp eq i32 %sel, %filt1_val - br i1 %is_filt1, label %handler1, label %eh.resume + to label %__try.cont unwind label %lpad0 + +lpad0: + %cs0 = catchswitch within none [label %handler0] unwind label %lpad1 handler0: - call void @puts(i8* getelementptr ([27 x i8], [27 x i8]* @str1, i32 0, i32 0)) + %p0 = catchpad within %cs0 [i8* bitcast (i32 (i8*, i8*)* @safe_div_filt0 to i8*)] + call void @puts(i8* getelementptr ([27 x i8], [27 x i8]* @str1, i32 0, i32 0)) [ "funclet"(token %p0) ] store i32 -1, i32* %r, align 4 - br label %__try.cont + catchret from %p0 to label %__try.cont + +lpad1: + %cs1 = catchswitch within none [label %handler1] unwind to caller handler1: - call void @puts(i8* getelementptr ([29 x i8], [29 x i8]* @str2, i32 0, i32 0)) + %p1 = catchpad within %cs1 [i8* bitcast (i32 (i8*, i8*)* @safe_div_filt1 to i8*)] + call void @puts(i8* getelementptr ([29 x i8], [29 x i8]* @str2, i32 0, i32 0)) [ "funclet"(token %p1) ] store i32 -2, i32* %r, align 4 - br label %__try.cont - -eh.resume: - resume { i8*, i32 } %vals + catchret from %p1 to label %__try.cont __try.cont: %safe_ret = load i32, i32* %r, align 4 @@ -68,7 +58,7 @@ __try.cont: ; CHECK: .seh_proc safe_div ; CHECK: .seh_handler __C_specific_handler, @unwind, @except ; CHECK: .Ltmp0: -; CHECK: leaq [[rloc:.*\(%rsp\)]], %rcx +; CHECK: leaq [[rloc:.*\(%rbp\)]], %rcx ; CHECK: callq try_body ; CHECK-NEXT: .Ltmp1 ; CHECK: [[cont_bb:\.LBB0_[0-9]+]]: @@ -77,32 +67,32 @@ __try.cont: ; Landing pad code -; CHECK: [[handler0:\.Ltmp[0-9]+]]: # Block address taken -; CHECK: # %handler0 +; CHECK: [[handler0:\.LBB0_[0-9]+]]: # %handler0 ; CHECK: callq puts ; CHECK: movl $-1, [[rloc]] ; CHECK: jmp [[cont_bb]] -; CHECK: [[handler1:\.Ltmp[0-9]+]]: # Block address taken -; CHECK: # %handler1 +; CHECK: [[handler1:\.LBB0_[0-9]+]]: # %handler1 ; CHECK: callq puts ; CHECK: movl $-2, [[rloc]] ; CHECK: jmp [[cont_bb]] ; CHECK: .seh_handlerdata -; CHECK-NEXT: .long 2 -; CHECK-NEXT: .long .Ltmp0@IMGREL +; CHECK-NEXT: .Lsafe_div$parent_frame_offset +; CHECK-NEXT: .long (.Llsda_end0-.Llsda_begin0)/16 +; CHECK-NEXT: .Llsda_begin0: +; CHECK-NEXT: .long .Ltmp0@IMGREL+1 ; CHECK-NEXT: .long .Ltmp1@IMGREL+1 ; CHECK-NEXT: .long safe_div_filt0@IMGREL ; CHECK-NEXT: .long [[handler0]]@IMGREL -; CHECK-NEXT: .long .Ltmp0@IMGREL +; CHECK-NEXT: .long .Ltmp0@IMGREL+1 ; CHECK-NEXT: .long .Ltmp1@IMGREL+1 ; CHECK-NEXT: .long safe_div_filt1@IMGREL ; CHECK-NEXT: .long [[handler1]]@IMGREL +; CHECK-NEXT: .Llsda_end0: ; CHECK: .text ; CHECK: .seh_endproc - define void @try_body(i32* %r, i32* %n, i32* %d) { entry: %0 = load i32, i32* %n, align 4 diff --git a/test/CodeGen/X86/seh-stack-realign-win32.ll b/test/CodeGen/X86/seh-stack-realign-win32.ll deleted file mode 100644 index f3ab71803ca7..000000000000 --- a/test/CodeGen/X86/seh-stack-realign-win32.ll +++ /dev/null @@ -1,99 +0,0 @@ -; RUN: llc -mtriple=i686-windows-msvc < %s | FileCheck %s - -; 32-bit catch-all has to use a filter function because that's how it saves the -; exception code. - -@str = linkonce_odr unnamed_addr constant [27 x i8] c"GetExceptionCode(): 0x%lx\0A\00", align 1 - -declare i32 @_except_handler3(...) -declare void @crash() -declare i32 @printf(i8* nocapture readonly, ...) nounwind -declare i32 @llvm.eh.typeid.for(i8*) -declare i8* @llvm.frameaddress(i32) -declare i8* @llvm.localrecover(i8*, i8*, i32) -declare void @llvm.localescape(...) -declare i8* @llvm.x86.seh.recoverfp(i8*, i8*) - -define i32 @main() personality i8* bitcast (i32 (...)* @_except_handler3 to i8*) { -entry: - ; The EH code allocation is overaligned, triggering realignment. - %__exceptioncode = alloca i32, align 8 - call void (...) @llvm.localescape(i32* %__exceptioncode) - invoke void @crash() #5 - to label %__try.cont unwind label %lpad - -lpad: ; preds = %entry - %0 = landingpad { i8*, i32 } - catch i8* bitcast (i32 ()* @"filt$main" to i8*) - %1 = extractvalue { i8*, i32 } %0, 1 - %2 = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @"filt$main" to i8*)) #4 - %matches = icmp eq i32 %1, %2 - br i1 %matches, label %__except, label %eh.resume - -__except: ; preds = %lpad - %3 = load i32, i32* %__exceptioncode, align 4 - %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([27 x i8], [27 x i8]* @str, i32 0, i32 0), i32 %3) #4 - br label %__try.cont - -__try.cont: ; preds = %entry, %__except - ret i32 0 - -eh.resume: ; preds = %lpad - resume { i8*, i32 } %0 -} - -define internal i32 @"filt$main"() { -entry: - %ebp = tail call i8* @llvm.frameaddress(i32 1) - %parentfp = tail call i8* @llvm.x86.seh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %ebp) - %code.i8 = tail call i8* @llvm.localrecover(i8* bitcast (i32 ()* @main to i8*), i8* %parentfp, i32 0) - %__exceptioncode = bitcast i8* %code.i8 to i32* - %info.addr = getelementptr inbounds i8, i8* %ebp, i32 -20 - %0 = bitcast i8* %info.addr to i32*** - %1 = load i32**, i32*** %0, align 4 - %2 = load i32*, i32** %1, align 4 - %3 = load i32, i32* %2, align 4 - store i32 %3, i32* %__exceptioncode, align 4 - ret i32 1 -} - -; Check that we can get the exception code from eax to the printf. - -; CHECK-LABEL: _main: -; CHECK: Lmain$frame_escape_0 = [[code_offs:[-0-9]+]] -; CHECK: Lmain$frame_escape_1 = [[reg_offs:[-0-9]+]] -; CHECK: movl %esp, [[reg_offs]](%esi) -; CHECK: movl $L__ehtable$main, -; EH state 0 -; CHECK: movl $0, 40(%esi) -; CHECK: calll _crash -; CHECK: retl -; CHECK: # Block address taken -; stackrestore -; CHECK: movl -24(%ebp), %esp -; CHECK: movl $Lmain$parent_frame_offset, %eax -; CHECK: negl %eax -; CHECK: leal -24(%ebp,%eax), %esi -; CHECK: movl 12(%esi), %ebp # 4-byte Reload -; EH state -1 -; CHECK: movl [[code_offs]](%esi), %[[code:[a-z]+]] -; CHECK: movl $-1, 40(%esi) -; CHECK-DAG: movl %[[code]], 4(%esp) -; CHECK-DAG: movl $_str, (%esp) -; CHECK: calll _printf - -; CHECK: .section .xdata,"dr" -; CHECK: Lmain$parent_frame_offset = Lmain$frame_escape_1 -; CHECK: L__ehtable$main -; CHECK-NEXT: .long -1 -; CHECK-NEXT: .long _filt$main -; CHECK-NEXT: .long Ltmp{{[0-9]+}} - -; CHECK-LABEL: _filt$main: -; CHECK: pushl %ebp -; CHECK: movl %esp, %ebp -; CHECK: movl (%ebp), %[[oldebp:[a-z]+]] -; CHECK: movl -20(%[[oldebp]]), %[[ehinfo:[a-z]+]] -; CHECK: movl (%[[ehinfo]]), %[[ehrec:[a-z]+]] -; CHECK: movl (%[[ehrec]]), %[[ehcode:[a-z]+]] -; CHECK: movl %[[ehcode]], {{.*}}(%{{.*}}) diff --git a/test/CodeGen/X86/seh-stack-realign.ll b/test/CodeGen/X86/seh-stack-realign.ll index f2fb28a081f9..654cad347f6b 100644 --- a/test/CodeGen/X86/seh-stack-realign.ll +++ b/test/CodeGen/X86/seh-stack-realign.ll @@ -23,23 +23,16 @@ entry: to label %__try.cont unwind label %lpad lpad: ; preds = %entry - %0 = landingpad { i8*, i32 } - catch i8* bitcast (i32 ()* @"filt$main" to i8*) - %1 = extractvalue { i8*, i32 } %0, 1 - %2 = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @"filt$main" to i8*)) #4 - %matches = icmp eq i32 %1, %2 - br i1 %matches, label %__except, label %eh.resume + %cs1 = catchswitch within none [label %__except] unwind to caller __except: ; preds = %lpad - %3 = load i32, i32* %__exceptioncode, align 4 - %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([27 x i8], [27 x i8]* @str, i32 0, i32 0), i32 %3) #4 - br label %__try.cont + %p = catchpad within %cs1 [i8* bitcast (i32 ()* @"filt$main" to i8*)] + %code = load i32, i32* %__exceptioncode, align 4 + %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([27 x i8], [27 x i8]* @str, i32 0, i32 0), i32 %code) #4 [ "funclet"(token %p) ] + catchret from %p to label %__try.cont __try.cont: ; preds = %entry, %__except ret i32 0 - -eh.resume: ; preds = %lpad - resume { i8*, i32 } %0 } define internal i32 @"filt$main"() { @@ -61,35 +54,30 @@ entry: ; CHECK-LABEL: _main: ; CHECK: Lmain$frame_escape_0 = [[code_offs:[-0-9]+]] -; CHECK: Lmain$frame_escape_1 = [[reg_offs:[-0-9]+]] -; CHECK: movl %esp, [[reg_offs]](%esi) +; CHECK: movl %esp, [[reg_offs:[-0-9]+]](%esi) ; CHECK: movl $L__ehtable$main, ; EH state 0 ; CHECK: movl $0, 40(%esi) ; CHECK: calll _crash ; CHECK: retl -; CHECK: # Block address taken +; CHECK: LBB0_[[lpbb:[0-9]+]]: # %__except ; Restore ESP ; CHECK: movl -24(%ebp), %esp ; Restore ESI -; CHECK: movl $Lmain$parent_frame_offset, %eax -; CHECK: negl %eax -; CHECK: leal -24(%ebp,%eax), %esi +; CHECK: leal -44(%ebp), %esi ; Restore EBP -; CHECK: movl 12(%esi), %ebp # 4-byte Reload -; EH state -1 +; CHECK: movl 12(%esi), %ebp ; CHECK: movl [[code_offs]](%esi), %[[code:[a-z]+]] -; CHECK: movl $-1, 40(%esi) ; CHECK-DAG: movl %[[code]], 4(%esp) ; CHECK-DAG: movl $_str, (%esp) ; CHECK: calll _printf ; CHECK: .section .xdata,"dr" -; CHECK: Lmain$parent_frame_offset = Lmain$frame_escape_1 +; CHECK: Lmain$parent_frame_offset = [[reg_offs]] ; CHECK: L__ehtable$main ; CHECK-NEXT: .long -1 ; CHECK-NEXT: .long _filt$main -; CHECK-NEXT: .long Ltmp{{[0-9]+}} +; CHECK-NEXT: .long LBB0_[[lpbb]] ; CHECK-LABEL: _filt$main: ; CHECK: pushl %ebp diff --git a/test/CodeGen/X86/setcc-lowering.ll b/test/CodeGen/X86/setcc-lowering.ll index 3149fb51576f..77739e72fcc8 100644 --- a/test/CodeGen/X86/setcc-lowering.ll +++ b/test/CodeGen/X86/setcc-lowering.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+avx < %s | FileCheck %s ; Verify that we don't crash during codegen due to a wrong lowering diff --git a/test/CodeGen/X86/setcc.ll b/test/CodeGen/X86/setcc.ll index 6f1ddbdc6aca..b4847c54ffaf 100644 --- a/test/CodeGen/X86/setcc.ll +++ b/test/CodeGen/X86/setcc.ll @@ -34,3 +34,23 @@ entry: %iftmp.2.0 = select i1 %0, i64 64, i64 0 ; [#uses=1] ret i64 %iftmp.2.0 } + +@v4 = common global i32 0, align 4 + +define i32 @t4(i32 %a) { +entry: +; CHECK-LABEL: t4: +; CHECK: movq _v4@GOTPCREL(%rip), %rax +; CHECK: cmpl $1, (%rax) +; CHECK: sbbl %eax, %eax +; CHECK: andl $32768, %eax +; CHECK: leal 65536(%rax,%rax), %eax + %0 = load i32, i32* @v4, align 4 + %not.tobool = icmp eq i32 %0, 0 + %conv.i = sext i1 %not.tobool to i16 + %call.lobit = lshr i16 %conv.i, 15 + %add.i.1 = add nuw nsw i16 %call.lobit, 1 + %conv4.2 = zext i16 %add.i.1 to i32 + %add = shl nuw nsw i32 %conv4.2, 16 + ret i32 %add +} diff --git a/test/CodeGen/X86/shift-bmi2.ll b/test/CodeGen/X86/shift-bmi2.ll index 63b6ec55fac8..fdeddffdfb0e 100644 --- a/test/CodeGen/X86/shift-bmi2.ll +++ b/test/CodeGen/X86/shift-bmi2.ll @@ -30,11 +30,10 @@ entry: %x = load i32, i32* %p %shl = shl i32 %x, %shamt ; BMI2: shl32p -; Source order scheduling prevents folding, rdar:14208996. -; BMI2: shlxl %{{.+}}, %{{.+}}, %{{.+}} +; BMI2: shlxl %{{.+}}, ({{.+}}), %{{.+}} ; BMI2: ret ; BMI264: shl32p -; BMI264: shlxl %{{.+}}, %{{.+}}, %{{.+}} +; BMI264: shlxl %{{.+}}, ({{.+}}), %{{.+}} ; BMI264: ret ret i32 %shl } @@ -75,7 +74,7 @@ entry: %x = load i64, i64* %p %shl = shl i64 %x, %shamt ; BMI264: shl64p -; BMI264: shlxq %{{.+}}, %{{.+}}, %{{.+}} +; BMI264: shlxq %{{.+}}, ({{.+}}), %{{.+}} ; BMI264: ret ret i64 %shl } @@ -107,11 +106,10 @@ entry: %x = load i32, i32* %p %shl = lshr i32 %x, %shamt ; BMI2: lshr32p -; Source order scheduling prevents folding, rdar:14208996. -; BMI2: shrxl %{{.+}}, %{{.+}}, %{{.+}} +; BMI2: shrxl %{{.+}}, ({{.+}}), %{{.+}} ; BMI2: ret ; BMI264: lshr32p -; BMI264: shrxl %{{.+}}, %{{.+}}, %{{.+}} +; BMI264: shrxl %{{.+}}, ({{.+}}), %{{.+}} ; BMI264: ret ret i32 %shl } @@ -130,7 +128,7 @@ entry: %x = load i64, i64* %p %shl = lshr i64 %x, %shamt ; BMI264: lshr64p -; BMI264: shrxq %{{.+}}, %{{.+}}, %{{.+}} +; BMI264: shrxq %{{.+}}, ({{.+}}), %{{.+}} ; BMI264: ret ret i64 %shl } @@ -153,10 +151,10 @@ entry: %shl = ashr i32 %x, %shamt ; BMI2: ashr32p ; Source order scheduling prevents folding, rdar:14208996. -; BMI2: sarxl %{{.+}}, %{{.+}}, %{{.+}} +; BMI2: sarxl %{{.+}}, ({{.+}}), %{{.+}} ; BMI2: ret ; BMI264: ashr32p -; BMI264: sarxl %{{.+}}, %{{.+}}, %{{.+}} +; BMI264: sarxl %{{.+}}, ({{.+}}), %{{.+}} ; BMI264: ret ret i32 %shl } @@ -175,7 +173,7 @@ entry: %x = load i64, i64* %p %shl = ashr i64 %x, %shamt ; BMI264: ashr64p -; BMI264: sarxq %{{.+}}, %{{.+}}, %{{.+}} +; BMI264: sarxq %{{.+}}, ({{.+}}), %{{.+}} ; BMI264: ret ret i64 %shl } diff --git a/test/CodeGen/X86/shrink-wrap-chkstk.ll b/test/CodeGen/X86/shrink-wrap-chkstk.ll new file mode 100644 index 000000000000..c0b2b45e676f --- /dev/null +++ b/test/CodeGen/X86/shrink-wrap-chkstk.ll @@ -0,0 +1,37 @@ +; RUN: llc < %s -enable-shrink-wrap=true | FileCheck %s + +; chkstk cannot come before the usual prologue, since it adjusts ESP. + +target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32" +target triple = "i686-pc-windows-msvc18.0.0" + +%struct.S = type { [12 x i8] } + +define x86_thiscallcc void @call_inalloca(i1 %x) { +entry: + %argmem = alloca inalloca <{ %struct.S }>, align 4 + %argidx1 = getelementptr inbounds <{ %struct.S }>, <{ %struct.S }>* %argmem, i32 0, i32 0, i32 0, i32 0 + %argidx2 = getelementptr inbounds <{ %struct.S }>, <{ %struct.S }>* %argmem, i32 0, i32 0, i32 0, i32 1 + store i8 42, i8* %argidx2, align 4 + br i1 %x, label %bb1, label %bb2 + +bb1: + store i8 42, i8* %argidx1, align 4 + br label %bb2 + +bb2: + call void @inalloca_params(<{ %struct.S }>* inalloca nonnull %argmem) + ret void +} + +; CHECK-LABEL: _call_inalloca: # @call_inalloca +; CHECK: pushl %ebp +; CHECK: movl %esp, %ebp +; CHECK: movl $12, %eax +; CHECK: calll __chkstk +; CHECK: calll _inalloca_params +; CHECK: movl %ebp, %esp +; CHECK: popl %ebp +; CHECK: retl + +declare void @inalloca_params(<{ %struct.S }>* inalloca) diff --git a/test/CodeGen/X86/slow-div.ll b/test/CodeGen/X86/slow-div.ll index 52223824bf96..82928521ac2b 100644 --- a/test/CodeGen/X86/slow-div.ll +++ b/test/CodeGen/X86/slow-div.ll @@ -25,4 +25,19 @@ entry: ret i64 %div } +; Verify that no extra code is generated when optimizing for size. + +define i32 @div32_optsize(i32 %a, i32 %b) optsize { +; DIV32-LABEL: div32_optsize: +; DIV32-NOT: divb + %div = sdiv i32 %a, %b + ret i32 %div +} + +define i32 @div32_minsize(i32 %a, i32 %b) minsize { +; DIV32-LABEL: div32_minsize: +; DIV32-NOT: divb + %div = sdiv i32 %a, %b + ret i32 %div +} diff --git a/test/CodeGen/X86/slow-unaligned-mem.ll b/test/CodeGen/X86/slow-unaligned-mem.ll new file mode 100644 index 000000000000..27cbef681b7e --- /dev/null +++ b/test/CodeGen/X86/slow-unaligned-mem.ll @@ -0,0 +1,95 @@ +; Intel chips with slow unaligned memory accesses + +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium3 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium3m 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium-m 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium4 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium4m 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=yonah 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=prescott 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=nocona 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=core2 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=penryn 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=bonnell 2>&1 | FileCheck %s --check-prefix=SLOW + +; Intel chips with fast unaligned memory accesses + +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=silvermont 2>&1 | FileCheck %s --check-prefix=FAST +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=nehalem 2>&1 | FileCheck %s --check-prefix=FAST +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=westmere 2>&1 | FileCheck %s --check-prefix=FAST +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=sandybridge 2>&1 | FileCheck %s --check-prefix=FAST +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=ivybridge 2>&1 | FileCheck %s --check-prefix=FAST +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=haswell 2>&1 | FileCheck %s --check-prefix=FAST +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=broadwell 2>&1 | FileCheck %s --check-prefix=FAST +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=knl 2>&1 | FileCheck %s --check-prefix=FAST +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=skylake 2>&1 | FileCheck %s --check-prefix=FAST + +; AMD chips with slow unaligned memory accesses + +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon-4 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon-xp 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=k8 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=opteron 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon64 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon-fx 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=k8-sse3 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=opteron-sse3 2>&1 | FileCheck %s --check-prefix=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon64-sse3 2>&1 | FileCheck %s --check-prefix=SLOW + +; AMD chips with fast unaligned memory accesses + +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=amdfam10 2>&1 | FileCheck %s --check-prefix=FAST +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=barcelona 2>&1 | FileCheck %s --check-prefix=FAST +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=btver1 2>&1 | FileCheck %s --check-prefix=FAST +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=btver2 2>&1 | FileCheck %s --check-prefix=FAST +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=bdver1 2>&1 | FileCheck %s --check-prefix=FAST +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=bdver2 2>&1 | FileCheck %s --check-prefix=FAST +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=bdver3 2>&1 | FileCheck %s --check-prefix=FAST +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=bdver4 2>&1 | FileCheck %s --check-prefix=FAST + +; Other chips with slow unaligned memory accesses + +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=c3-2 2>&1 | FileCheck %s --check-prefix=SLOW + +; Verify that the slow/fast unaligned memory attribute is set correctly for each CPU model. +; Slow chips use 4-byte stores. Fast chips with SSE or later use something other than 4-byte stores. +; Chips that don't have SSE use 4-byte stores either way, so they're not tested. + +; Also verify that SSE4.2 or SSE4a imply fast unaligned accesses. + +; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=sse4.2 2>&1 | FileCheck %s --check-prefix=FAST +; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=sse4a 2>&1 | FileCheck %s --check-prefix=FAST + +define void @store_zeros(i8* %a) { +; SLOW-NOT: not a recognized processor +; SLOW-LABEL: store_zeros: +; SLOW: # BB#0: +; SLOW-NEXT: movl +; SLOW-NEXT: movl +; SLOW-NEXT: movl +; SLOW-NEXT: movl +; SLOW-NEXT: movl +; SLOW-NEXT: movl +; SLOW-NEXT: movl +; SLOW-NEXT: movl +; SLOW-NEXT: movl +; SLOW-NEXT: movl +; SLOW-NEXT: movl +; SLOW-NEXT: movl +; SLOW-NEXT: movl +; SLOW-NEXT: movl +; SLOW-NEXT: movl +; SLOW-NEXT: movl +; SLOW-NEXT: movl +; +; FAST-NOT: not a recognized processor +; FAST-LABEL: store_zeros: +; FAST: # BB#0: +; FAST-NEXT: movl {{[0-9]+}}(%esp), %eax +; FAST-NOT: movl + call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 64, i32 1, i1 false) + ret void +} + +declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i32, i1) + diff --git a/test/CodeGen/X86/soft-fp.ll b/test/CodeGen/X86/soft-fp.ll index fa38d1044a48..138e66c394ba 100644 --- a/test/CodeGen/X86/soft-fp.ll +++ b/test/CodeGen/X86/soft-fp.ll @@ -1,7 +1,14 @@ -; RUN: llc < %s -march=x86 -mattr=+sse2,+soft-float | FileCheck %s -; RUN: llc < %s -march=x86-64 -mattr=+sse2,+soft-float | FileCheck %s +; RUN: llc < %s -march=x86 -mattr=+mmx,+sse,+soft-float \ +; RUN: | FileCheck %s --check-prefix=SOFT1 --check-prefix=CHECK +; RUN: llc < %s -march=x86-64 -mattr=+mmx,+sse2,+soft-float \ +; RUN: | FileCheck %s --check-prefix=SOFT2 --check-prefix=CHECK +; RUN: llc < %s -march=x86-64 -mattr=+mmx,+sse \ +; RUN: | FileCheck %s --check-prefix=SSE1 --check-prefix=CHECK +; RUN: llc < %s -march=x86-64 -mattr=+mmx,+sse2 \ +; RUN: | FileCheck %s --check-prefix=SSE2 --check-prefix=CHECK +; RUN: llc < %s -mtriple=x86_64-gnux32 -mattr=+mmx,+sse2,+soft-float | FileCheck %s -; CHECK-NOT: xmm{[0-9]+} +; CHECK-NOT: xmm{{[0-9]+}} %struct.__va_list_tag = type { i32, i32, i8*, i8* } @@ -14,6 +21,8 @@ entry: call void @bar(%struct.__va_list_tag* %va3) nounwind call void @llvm.va_end(i8* %va12) ret i32 undef +; CHECK-LABEL: t1: +; CHECK: ret{{[lq]}} } declare void @llvm.va_start(i8*) nounwind @@ -26,4 +35,23 @@ define float @t2(float %a, float %b) nounwind readnone { entry: %0 = fadd float %a, %b ; [#uses=1] ret float %0 +; CHECK-LABEL: t2: +; SOFT1-NOT: xmm{{[0-9]+}} +; SOFT2-NOT: xmm{{[0-9]+}} +; SSE1: xmm{{[0-9]+}} +; SSE2: xmm{{[0-9]+}} +; CHECK: ret{{[lq]}} +} + +; soft-float means no SSE instruction and passing fp128 as pair of i64. +define fp128 @t3(fp128 %a, fp128 %b) nounwind readnone { +entry: + %0 = fadd fp128 %b, %a + ret fp128 %0 +; CHECK-LABEL: t3: +; SOFT1-NOT: xmm{{[0-9]+}} +; SOFT2-NOT: xmm{{[0-9]+}} +; SSE1: xmm{{[0-9]+}} +; SSE2: xmm{{[0-9]+}} +; CHECK: ret{{[lq]}} } diff --git a/test/CodeGen/X86/soft-sitofp.ll b/test/CodeGen/X86/soft-sitofp.ll new file mode 100644 index 000000000000..acb4bb906e70 --- /dev/null +++ b/test/CodeGen/X86/soft-sitofp.ll @@ -0,0 +1,169 @@ +; RUN: llc -mtriple=i386-pc-linux < %s | FileCheck %s +; RUN: llc -mtriple=x86_64-pc-linux < %s | FileCheck %s + +; Function Attrs: nounwind +; CHECK-LABEL: s64_to_d: +; CHECK: call{{l|q}} __floatdidf +define double @s64_to_d(i64 %n) #0 { +entry: + %conv = sitofp i64 %n to double + ret double %conv +} + +; CHECK-LABEL: s64_to_f: +; CHECK: call{{l|q}} __floatdisf +define float @s64_to_f(i64 %n) #0 { +entry: + %conv = sitofp i64 %n to float + ret float %conv +} + +; CHECK-LABEL: s32_to_d: +; CHECK: call{{l|q}} __floatsidf +define double @s32_to_d(i32 %n) #0 { +entry: + %conv = sitofp i32 %n to double + ret double %conv +} + +; CHECK-LABEL: s32_to_f: +; CHECK: call{{l|q}} __floatsisf +define float @s32_to_f(i32 %n) #0 { +entry: + %conv = sitofp i32 %n to float + ret float %conv +} + +; CHECK-LABEL: u64_to_d: +; CHECK: call{{l|q}} __floatundidf +define double @u64_to_d(i64 %n) #0 { +entry: + %conv = uitofp i64 %n to double + ret double %conv +} + +; CHECK-LABEL: u64_to_f: +; CHECK: call{{l|q}} __floatundisf +define float @u64_to_f(i64 %n) #0 { +entry: + %conv = uitofp i64 %n to float + ret float %conv +} + +; CHECK-LABEL: u32_to_d: +; CHECK: call{{l|q}} __floatunsidf +define double @u32_to_d(i32 %n) #0 { +entry: + %conv = uitofp i32 %n to double + ret double %conv +} + +; CHECK-LABEL: u32_to_f: +; CHECK: call{{l|q}} __floatunsisf +define float @u32_to_f(i32 %n) #0 { +entry: + %conv = uitofp i32 %n to float + ret float %conv +} + +; CHECK-LABEL: d_to_s64: +; CHECK: call{{l|q}} __fixdfdi +define i64 @d_to_s64(double %n) #0 { +entry: + %conv = fptosi double %n to i64 + ret i64 %conv +} + +; CHECK-LABEL: d_to_s32: +; CHECK: call{{l|q}} __fixdfsi +define i32 @d_to_s32(double %n) #0 { +entry: + %conv = fptosi double %n to i32 + ret i32 %conv +} + +; CHECK-LABEL: f_to_s64: +; CHECK: call{{l|q}} __fixsfdi +define i64 @f_to_s64(float %n) #0 { +entry: + %conv = fptosi float %n to i64 + ret i64 %conv +} + +; CHECK-LABEL: f_to_s32: +; CHECK: call{{l|q}} __fixsfsi +define i32 @f_to_s32(float %n) #0 { +entry: + %conv = fptosi float %n to i32 + ret i32 %conv +} + +; CHECK-LABEL: d_to_u64: +; CHECK: call{{l|q}} __fixunsdfdi +define i64 @d_to_u64(double %n) #0 { +entry: + %conv = fptoui double %n to i64 + ret i64 %conv +} + +; CHECK-LABEL: d_to_u32: +; CHECK: call{{l|q}} __fixunsdfsi +define i32 @d_to_u32(double %n) #0 { +entry: + %conv = fptoui double %n to i32 + ret i32 %conv +} + +; CHECK-LABEL: f_to_u64: +; CHECK: call{{l|q}} __fixunssfdi +define i64 @f_to_u64(float %n) #0 { +entry: + %conv = fptoui float %n to i64 + ret i64 %conv +} + +; CHECK-LABEL: f_to_u32: +; CHECK: call{{l|q}} __fixunssfsi +define i32 @f_to_u32(float %n) #0 { +entry: + %conv = fptoui float %n to i32 + ret i32 %conv +} + +; CHECK-LABEL: f_to_s8: +; CHECK: call{{l|q}} __fixsfsi +define i8 @f_to_s8(float %f, i8 %i) #0 { +entry: + %conv = fptosi float %f to i8 + %add = add i8 %conv, %i + ret i8 %add +} + +; CHECK-LABEL: f_to_u8: +; CHECK: call{{l|q}} __fixunssfsi +define i8 @f_to_u8(float %f, i8 %i) #0 { +entry: + %conv = fptoui float %f to i8 + %add = add i8 %conv, %i + ret i8 %add +} + +; CHECK-LABEL: f_to_s16: +; CHECK: call{{l|q}} __fixsfsi +define i16 @f_to_s16(float %f, i16 %i) #0 { +entry: + %conv = fptosi float %f to i16 + %add = add i16 %conv, %i + ret i16 %add +} + +; CHECK-LABEL: f_to_u16: +; CHECK: call{{l|q}} __fixunssfsi +define i16 @f_to_u16(float %f, i16 %i) #0 { +entry: + %conv = fptoui float %f to i16 + %add = add i16 %conv, %i + ret i16 %add +} + +attributes #0 = { nounwind "use-soft-float"="true" } diff --git a/test/CodeGen/X86/splat-for-size.ll b/test/CodeGen/X86/splat-for-size.ll index 635aa821d78a..277472f49b3a 100644 --- a/test/CodeGen/X86/splat-for-size.ll +++ b/test/CodeGen/X86/splat-for-size.ll @@ -1,141 +1,191 @@ -; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=avx < %s | FileCheck %s -check-prefix=CHECK --check-prefix=AVX -; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=avx2 < %s | FileCheck %s -check-prefix=CHECK --check-prefix=AVX2 +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=CHECK --check-prefix=AVX +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=AVX2 -; Check constant loads of every 128-bit and 256-bit vector type +; Check constant loads of every 128-bit and 256-bit vector type ; for size optimization using splat ops available with AVX and AVX2. ; There is no AVX broadcast from double to 128-bit vector because movddup has been around since SSE3 (grrr). define <2 x double> @splat_v2f64(<2 x double> %x) #0 { +; CHECK-LABEL: splat_v2f64: +; CHECK: # BB#0: +; CHECK-NEXT: vmovddup {{.*#+}} xmm1 = mem[0,0] +; CHECK-NEXT: vaddpd %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: retq %add = fadd <2 x double> %x, ret <2 x double> %add -; CHECK-LABEL: splat_v2f64 -; CHECK: vmovddup -; CHECK: vaddpd -; CHECK-NEXT: retq } -define <4 x double> @splat_v4f64(<4 x double> %x) #0 { +define <4 x double> @splat_v4f64(<4 x double> %x) #1 { +; CHECK-LABEL: splat_v4f64: +; CHECK: # BB#0: +; CHECK-NEXT: vbroadcastsd {{.*}}(%rip), %ymm1 +; CHECK-NEXT: vaddpd %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: retq %add = fadd <4 x double> %x, ret <4 x double> %add -; CHECK-LABEL: splat_v4f64 -; CHECK: vbroadcastsd -; CHECK-NEXT: vaddpd -; CHECK-NEXT: retq } define <4 x float> @splat_v4f32(<4 x float> %x) #0 { +; CHECK-LABEL: splat_v4f32: +; CHECK: # BB#0: +; CHECK-NEXT: vbroadcastss {{.*}}(%rip), %xmm1 +; CHECK-NEXT: vaddps %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: retq %add = fadd <4 x float> %x, ret <4 x float> %add -; CHECK-LABEL: splat_v4f32 -; CHECK: vbroadcastss -; CHECK-NEXT: vaddps -; CHECK-NEXT: retq } -define <8 x float> @splat_v8f32(<8 x float> %x) #0 { +define <8 x float> @splat_v8f32(<8 x float> %x) #1 { +; CHECK-LABEL: splat_v8f32: +; CHECK: # BB#0: +; CHECK-NEXT: vbroadcastss {{.*}}(%rip), %ymm1 +; CHECK-NEXT: vaddps %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: retq %add = fadd <8 x float> %x, ret <8 x float> %add -; CHECK-LABEL: splat_v8f32 -; CHECK: vbroadcastss -; CHECK-NEXT: vaddps -; CHECK-NEXT: retq } ; AVX can't do integer splats, so fake it: use vmovddup to splat 64-bit value. ; We also generate vmovddup for AVX2 because it's one byte smaller than vpbroadcastq. -define <2 x i64> @splat_v2i64(<2 x i64> %x) #0 { +define <2 x i64> @splat_v2i64(<2 x i64> %x) #1 { +; CHECK-LABEL: splat_v2i64: +; CHECK: # BB#0: +; CHECK-NEXT: vmovddup {{.*#+}} xmm1 = mem[0,0] +; CHECK-NEXT: vpaddq %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: retq %add = add <2 x i64> %x, ret <2 x i64> %add -; CHECK-LABEL: splat_v2i64 -; CHECK: vmovddup -; CHECK: vpaddq -; CHECK-NEXT: retq } ; AVX can't do 256-bit integer ops, so we split this into two 128-bit vectors, ; and then we fake it: use vmovddup to splat 64-bit value. define <4 x i64> @splat_v4i64(<4 x i64> %x) #0 { +; AVX-LABEL: splat_v4i64: +; AVX: # BB#0: +; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX-NEXT: vmovddup {{.*#+}} xmm2 = mem[0,0] +; AVX-NEXT: vpaddq %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpaddq %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX-NEXT: retq +; +; AVX2-LABEL: splat_v4i64: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm1 +; AVX2-NEXT: vpaddq %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq %add = add <4 x i64> %x, ret <4 x i64> %add -; CHECK-LABEL: splat_v4i64 -; AVX: vmovddup -; AVX: vpaddq -; AVX: vpaddq -; AVX2: vpbroadcastq -; AVX2: vpaddq -; CHECK: retq } ; AVX can't do integer splats, so fake it: use vbroadcastss to splat 32-bit value. -define <4 x i32> @splat_v4i32(<4 x i32> %x) #0 { +define <4 x i32> @splat_v4i32(<4 x i32> %x) #1 { +; AVX-LABEL: splat_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vbroadcastss {{.*}}(%rip), %xmm1 +; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; AVX2-LABEL: splat_v4i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %xmm1 +; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq %add = add <4 x i32> %x, ret <4 x i32> %add -; CHECK-LABEL: splat_v4i32 -; AVX: vbroadcastss -; AVX2: vpbroadcastd -; CHECK-NEXT: vpaddd -; CHECK-NEXT: retq } ; AVX can't do integer splats, so fake it: use vbroadcastss to splat 32-bit value. define <8 x i32> @splat_v8i32(<8 x i32> %x) #0 { +; AVX-LABEL: splat_v8i32: +; AVX: # BB#0: +; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX-NEXT: vbroadcastss {{.*}}(%rip), %xmm2 +; AVX-NEXT: vpaddd %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpaddd %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX-NEXT: retq +; +; AVX2-LABEL: splat_v8i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %ymm1 +; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq %add = add <8 x i32> %x, ret <8 x i32> %add -; CHECK-LABEL: splat_v8i32 -; AVX: vbroadcastss -; AVX: vpaddd -; AVX: vpaddd -; AVX2: vpbroadcastd -; AVX2: vpaddd -; CHECK: retq } ; AVX can't do integer splats, and there's no broadcast fakery for 16-bit. Could use pshuflw, etc? -define <8 x i16> @splat_v8i16(<8 x i16> %x) #0 { +define <8 x i16> @splat_v8i16(<8 x i16> %x) #1 { +; AVX-LABEL: splat_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpaddw {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: retq +; +; AVX2-LABEL: splat_v8i16: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastw {{.*}}(%rip), %xmm1 +; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq %add = add <8 x i16> %x, ret <8 x i16> %add -; CHECK-LABEL: splat_v8i16 -; AVX-NOT: broadcast -; AVX2: vpbroadcastw -; CHECK: vpaddw -; CHECK-NEXT: retq } ; AVX can't do integer splats, and there's no broadcast fakery for 16-bit. Could use pshuflw, etc? define <16 x i16> @splat_v16i16(<16 x i16> %x) #0 { +; AVX-LABEL: splat_v16i16: +; AVX: # BB#0: +; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1] +; AVX-NEXT: vpaddw %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpaddw %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX-NEXT: retq +; +; AVX2-LABEL: splat_v16i16: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastw {{.*}}(%rip), %ymm1 +; AVX2-NEXT: vpaddw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq %add = add <16 x i16> %x, ret <16 x i16> %add -; CHECK-LABEL: splat_v16i16 -; AVX-NOT: broadcast -; AVX: vpaddw -; AVX: vpaddw -; AVX2: vpbroadcastw -; AVX2: vpaddw -; CHECK: retq } ; AVX can't do integer splats, and there's no broadcast fakery for 8-bit. Could use pshufb, etc? -define <16 x i8> @splat_v16i8(<16 x i8> %x) #0 { +define <16 x i8> @splat_v16i8(<16 x i8> %x) #1 { +; AVX-LABEL: splat_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpaddb {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: retq +; +; AVX2-LABEL: splat_v16i8: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastb {{.*}}(%rip), %xmm1 +; AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq %add = add <16 x i8> %x, ret <16 x i8> %add -; CHECK-LABEL: splat_v16i8 -; AVX-NOT: broadcast -; AVX2: vpbroadcastb -; CHECK: vpaddb -; CHECK-NEXT: retq } ; AVX can't do integer splats, and there's no broadcast fakery for 8-bit. Could use pshufb, etc? define <32 x i8> @splat_v32i8(<32 x i8> %x) #0 { +; AVX-LABEL: splat_v32i8: +; AVX: # BB#0: +; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; AVX-NEXT: vpaddb %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpaddb %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX-NEXT: retq +; +; AVX2-LABEL: splat_v32i8: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastb {{.*}}(%rip), %ymm1 +; AVX2-NEXT: vpaddb %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq %add = add <32 x i8> %x, ret <32 x i8> %add -; CHECK-LABEL: splat_v32i8 -; AVX-NOT: broadcast -; AVX: vpaddb -; AVX: vpaddb -; AVX2: vpbroadcastb -; AVX2: vpaddb -; CHECK: retq } ; PR23259: Verify that ISel doesn't crash with a 'fatal error in backend' @@ -144,7 +194,7 @@ define <32 x i8> @splat_v32i8(<32 x i8> %x) #0 { @A = common global <3 x i64> zeroinitializer, align 32 -define <8 x i64> @pr23259() #0 { +define <8 x i64> @pr23259() #1 { entry: %0 = load <4 x i64>, <4 x i64>* bitcast (<3 x i64>* @A to <4 x i64>*), align 32 %1 = shufflevector <4 x i64> %0, <4 x i64> undef, <3 x i32> @@ -153,3 +203,4 @@ entry: } attributes #0 = { optsize } +attributes #1 = { minsize } diff --git a/test/CodeGen/X86/sqrt-fastmath.ll b/test/CodeGen/X86/sqrt-fastmath.ll index 9b851db8121c..386409a674ef 100644 --- a/test/CodeGen/X86/sqrt-fastmath.ll +++ b/test/CodeGen/X86/sqrt-fastmath.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse2 -recip=!sqrtf,!vec-sqrtf,!divf,!vec-divf | FileCheck %s --check-prefix=NORECIP ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx -recip=sqrtf,vec-sqrtf | FileCheck %s --check-prefix=ESTIMATE @@ -99,8 +100,8 @@ define <4 x float> @reciprocal_square_root_v4f32(<4 x float> %x) #0 { ; ESTIMATE-LABEL: reciprocal_square_root_v4f32: ; ESTIMATE: # BB#0: ; ESTIMATE-NEXT: vrsqrtps %xmm0, %xmm1 -; ESTIMATE-NEXT: vmulps %xmm1, %xmm1, %xmm2 -; ESTIMATE-NEXT: vmulps %xmm0, %xmm2, %xmm0 +; ESTIMATE-NEXT: vmulps %xmm0, %xmm1, %xmm0 +; ESTIMATE-NEXT: vmulps %xmm0, %xmm1, %xmm0 ; ESTIMATE-NEXT: vaddps {{.*}}(%rip), %xmm0, %xmm0 ; ESTIMATE-NEXT: vmulps {{.*}}(%rip), %xmm1, %xmm1 ; ESTIMATE-NEXT: vmulps %xmm1, %xmm0, %xmm0 @@ -124,8 +125,8 @@ define <8 x float> @reciprocal_square_root_v8f32(<8 x float> %x) #0 { ; ESTIMATE-LABEL: reciprocal_square_root_v8f32: ; ESTIMATE: # BB#0: ; ESTIMATE-NEXT: vrsqrtps %ymm0, %ymm1 -; ESTIMATE-NEXT: vmulps %ymm1, %ymm1, %ymm2 -; ESTIMATE-NEXT: vmulps %ymm0, %ymm2, %ymm0 +; ESTIMATE-NEXT: vmulps %ymm0, %ymm1, %ymm0 +; ESTIMATE-NEXT: vmulps %ymm0, %ymm1, %ymm0 ; ESTIMATE-NEXT: vaddps {{.*}}(%rip), %ymm0, %ymm0 ; ESTIMATE-NEXT: vmulps {{.*}}(%rip), %ymm1, %ymm1 ; ESTIMATE-NEXT: vmulps %ymm1, %ymm0, %ymm0 diff --git a/test/CodeGen/X86/sse-align-12.ll b/test/CodeGen/X86/sse-align-12.ll index 9441cc0002fb..4fbb6e42ccae 100644 --- a/test/CodeGen/X86/sse-align-12.ll +++ b/test/CodeGen/X86/sse-align-12.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -march=x86-64 -mcpu=nehalem | FileCheck %s define <4 x float> @a(<4 x float>* %y) nounwind { diff --git a/test/CodeGen/X86/sse-minmax.ll b/test/CodeGen/X86/sse-minmax.ll index e4d0373299fb..f0341277851d 100644 --- a/test/CodeGen/X86/sse-minmax.ll +++ b/test/CodeGen/X86/sse-minmax.ll @@ -3,7 +3,7 @@ ; RUN: llc < %s -march=x86-64 -mtriple=x86_64-apple-darwin -mcpu=nehalem -asm-verbose=false -enable-no-nans-fp-math | FileCheck -check-prefix=FINITE %s ; Some of these patterns can be matched as SSE min or max. Some of -; then can be matched provided that the operands are swapped. +; them can be matched provided that the operands are swapped. ; Some of them can't be matched at all and require a comparison ; and a conditional branch. diff --git a/test/CodeGen/X86/sse-only.ll b/test/CodeGen/X86/sse-only.ll new file mode 100644 index 000000000000..3fe9faaba850 --- /dev/null +++ b/test/CodeGen/X86/sse-only.ll @@ -0,0 +1,20 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -march=x86 -mattr=+sse2,-mmx | FileCheck %s + +; Test that turning off mmx doesn't turn off sse + +define void @test1(<2 x double>* %r, <2 x double>* %A, double %B) nounwind { +; CHECK-LABEL: test1: +; CHECK: # BB#0: +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx +; CHECK-NEXT: movapd (%ecx), %xmm0 +; CHECK-NEXT: movlpd {{[0-9]+}}(%esp), %xmm0 +; CHECK-NEXT: movapd %xmm0, (%eax) +; CHECK-NEXT: retl + %tmp3 = load <2 x double>, <2 x double>* %A, align 16 + %tmp7 = insertelement <2 x double> undef, double %B, i32 0 + %tmp9 = shufflevector <2 x double> %tmp3, <2 x double> %tmp7, <2 x i32> < i32 2, i32 1 > + store <2 x double> %tmp9, <2 x double>* %r, align 16 + ret void +} diff --git a/test/CodeGen/X86/sse-scalar-fp-arith-unary.ll b/test/CodeGen/X86/sse-scalar-fp-arith-unary.ll index fab4f90279e8..63751e1ab7e1 100644 --- a/test/CodeGen/X86/sse-scalar-fp-arith-unary.ll +++ b/test/CodeGen/X86/sse-scalar-fp-arith-unary.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=sse2 < %s | FileCheck --check-prefix=SSE %s ; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=sse4.1 < %s | FileCheck --check-prefix=SSE %s ; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=avx < %s | FileCheck --check-prefix=AVX %s diff --git a/test/CodeGen/X86/sse2-vector-shifts.ll b/test/CodeGen/X86/sse2-vector-shifts.ll index 45028cf4bd37..d1c7adb6263b 100644 --- a/test/CodeGen/X86/sse2-vector-shifts.ll +++ b/test/CodeGen/X86/sse2-vector-shifts.ll @@ -1,367 +1,373 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+sse2 | FileCheck %s ; SSE2 Logical Shift Left define <8 x i16> @test_sllw_1(<8 x i16> %InVec) { +; CHECK-LABEL: test_sllw_1: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: retq entry: %shl = shl <8 x i16> %InVec, ret <8 x i16> %shl } -; CHECK-LABEL: test_sllw_1: -; CHECK-NOT: psllw $0, %xmm0 -; CHECK: ret - define <8 x i16> @test_sllw_2(<8 x i16> %InVec) { +; CHECK-LABEL: test_sllw_2: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: paddw %xmm0, %xmm0 +; CHECK-NEXT: retq entry: %shl = shl <8 x i16> %InVec, ret <8 x i16> %shl } -; CHECK-LABEL: test_sllw_2: -; CHECK: paddw %xmm0, %xmm0 -; CHECK-NEXT: ret - define <8 x i16> @test_sllw_3(<8 x i16> %InVec) { +; CHECK-LABEL: test_sllw_3: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: psllw $15, %xmm0 +; CHECK-NEXT: retq entry: %shl = shl <8 x i16> %InVec, ret <8 x i16> %shl } -; CHECK-LABEL: test_sllw_3: -; CHECK: psllw $15, %xmm0 -; CHECK-NEXT: ret - define <4 x i32> @test_slld_1(<4 x i32> %InVec) { +; CHECK-LABEL: test_slld_1: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: retq entry: %shl = shl <4 x i32> %InVec, ret <4 x i32> %shl } -; CHECK-LABEL: test_slld_1: -; CHECK-NOT: pslld $0, %xmm0 -; CHECK: ret - define <4 x i32> @test_slld_2(<4 x i32> %InVec) { +; CHECK-LABEL: test_slld_2: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: paddd %xmm0, %xmm0 +; CHECK-NEXT: retq entry: %shl = shl <4 x i32> %InVec, ret <4 x i32> %shl } -; CHECK-LABEL: test_slld_2: -; CHECK: paddd %xmm0, %xmm0 -; CHECK-NEXT: ret - define <4 x i32> @test_slld_3(<4 x i32> %InVec) { +; CHECK-LABEL: test_slld_3: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: pslld $31, %xmm0 +; CHECK-NEXT: retq entry: %shl = shl <4 x i32> %InVec, ret <4 x i32> %shl } -; CHECK-LABEL: test_slld_3: -; CHECK: pslld $31, %xmm0 -; CHECK-NEXT: ret - define <2 x i64> @test_sllq_1(<2 x i64> %InVec) { +; CHECK-LABEL: test_sllq_1: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: retq entry: %shl = shl <2 x i64> %InVec, ret <2 x i64> %shl } -; CHECK-LABEL: test_sllq_1: -; CHECK-NOT: psllq $0, %xmm0 -; CHECK: ret - define <2 x i64> @test_sllq_2(<2 x i64> %InVec) { +; CHECK-LABEL: test_sllq_2: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: paddq %xmm0, %xmm0 +; CHECK-NEXT: retq entry: %shl = shl <2 x i64> %InVec, ret <2 x i64> %shl } -; CHECK-LABEL: test_sllq_2: -; CHECK: paddq %xmm0, %xmm0 -; CHECK-NEXT: ret - define <2 x i64> @test_sllq_3(<2 x i64> %InVec) { +; CHECK-LABEL: test_sllq_3: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: psllq $63, %xmm0 +; CHECK-NEXT: retq entry: %shl = shl <2 x i64> %InVec, ret <2 x i64> %shl } -; CHECK-LABEL: test_sllq_3: -; CHECK: psllq $63, %xmm0 -; CHECK-NEXT: ret - ; SSE2 Arithmetic Shift define <8 x i16> @test_sraw_1(<8 x i16> %InVec) { +; CHECK-LABEL: test_sraw_1: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: retq entry: %shl = ashr <8 x i16> %InVec, ret <8 x i16> %shl } -; CHECK-LABEL: test_sraw_1: -; CHECK-NOT: psraw $0, %xmm0 -; CHECK: ret - define <8 x i16> @test_sraw_2(<8 x i16> %InVec) { +; CHECK-LABEL: test_sraw_2: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: psraw $1, %xmm0 +; CHECK-NEXT: retq entry: %shl = ashr <8 x i16> %InVec, ret <8 x i16> %shl } -; CHECK-LABEL: test_sraw_2: -; CHECK: psraw $1, %xmm0 -; CHECK-NEXT: ret - define <8 x i16> @test_sraw_3(<8 x i16> %InVec) { +; CHECK-LABEL: test_sraw_3: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: psraw $15, %xmm0 +; CHECK-NEXT: retq entry: %shl = ashr <8 x i16> %InVec, ret <8 x i16> %shl } -; CHECK-LABEL: test_sraw_3: -; CHECK: psraw $15, %xmm0 -; CHECK-NEXT: ret - define <4 x i32> @test_srad_1(<4 x i32> %InVec) { +; CHECK-LABEL: test_srad_1: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: retq entry: %shl = ashr <4 x i32> %InVec, ret <4 x i32> %shl } -; CHECK-LABEL: test_srad_1: -; CHECK-NOT: psrad $0, %xmm0 -; CHECK: ret - define <4 x i32> @test_srad_2(<4 x i32> %InVec) { +; CHECK-LABEL: test_srad_2: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: psrad $1, %xmm0 +; CHECK-NEXT: retq entry: %shl = ashr <4 x i32> %InVec, ret <4 x i32> %shl } -; CHECK-LABEL: test_srad_2: -; CHECK: psrad $1, %xmm0 -; CHECK-NEXT: ret - define <4 x i32> @test_srad_3(<4 x i32> %InVec) { +; CHECK-LABEL: test_srad_3: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: psrad $31, %xmm0 +; CHECK-NEXT: retq entry: %shl = ashr <4 x i32> %InVec, ret <4 x i32> %shl } -; CHECK-LABEL: test_srad_3: -; CHECK: psrad $31, %xmm0 -; CHECK-NEXT: ret - ; SSE Logical Shift Right define <8 x i16> @test_srlw_1(<8 x i16> %InVec) { +; CHECK-LABEL: test_srlw_1: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: retq entry: %shl = lshr <8 x i16> %InVec, ret <8 x i16> %shl } -; CHECK-LABEL: test_srlw_1: -; CHECK-NOT: psrlw $0, %xmm0 -; CHECK: ret - define <8 x i16> @test_srlw_2(<8 x i16> %InVec) { +; CHECK-LABEL: test_srlw_2: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: psrlw $1, %xmm0 +; CHECK-NEXT: retq entry: %shl = lshr <8 x i16> %InVec, ret <8 x i16> %shl } -; CHECK-LABEL: test_srlw_2: -; CHECK: psrlw $1, %xmm0 -; CHECK-NEXT: ret - define <8 x i16> @test_srlw_3(<8 x i16> %InVec) { +; CHECK-LABEL: test_srlw_3: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: psrlw $15, %xmm0 +; CHECK-NEXT: retq entry: %shl = lshr <8 x i16> %InVec, ret <8 x i16> %shl } -; CHECK-LABEL: test_srlw_3: -; CHECK: psrlw $15, %xmm0 -; CHECK-NEXT: ret - define <4 x i32> @test_srld_1(<4 x i32> %InVec) { +; CHECK-LABEL: test_srld_1: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: retq entry: %shl = lshr <4 x i32> %InVec, ret <4 x i32> %shl } -; CHECK-LABEL: test_srld_1: -; CHECK-NOT: psrld $0, %xmm0 -; CHECK: ret - define <4 x i32> @test_srld_2(<4 x i32> %InVec) { +; CHECK-LABEL: test_srld_2: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: psrld $1, %xmm0 +; CHECK-NEXT: retq entry: %shl = lshr <4 x i32> %InVec, ret <4 x i32> %shl } -; CHECK-LABEL: test_srld_2: -; CHECK: psrld $1, %xmm0 -; CHECK-NEXT: ret - define <4 x i32> @test_srld_3(<4 x i32> %InVec) { +; CHECK-LABEL: test_srld_3: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: psrld $31, %xmm0 +; CHECK-NEXT: retq entry: %shl = lshr <4 x i32> %InVec, ret <4 x i32> %shl } -; CHECK-LABEL: test_srld_3: -; CHECK: psrld $31, %xmm0 -; CHECK-NEXT: ret - define <2 x i64> @test_srlq_1(<2 x i64> %InVec) { +; CHECK-LABEL: test_srlq_1: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: retq entry: %shl = lshr <2 x i64> %InVec, ret <2 x i64> %shl } -; CHECK-LABEL: test_srlq_1: -; CHECK-NOT: psrlq $0, %xmm0 -; CHECK: ret - define <2 x i64> @test_srlq_2(<2 x i64> %InVec) { +; CHECK-LABEL: test_srlq_2: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: psrlq $1, %xmm0 +; CHECK-NEXT: retq entry: %shl = lshr <2 x i64> %InVec, ret <2 x i64> %shl } -; CHECK-LABEL: test_srlq_2: -; CHECK: psrlq $1, %xmm0 -; CHECK-NEXT: ret - define <2 x i64> @test_srlq_3(<2 x i64> %InVec) { +; CHECK-LABEL: test_srlq_3: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: psrlq $63, %xmm0 +; CHECK-NEXT: retq entry: %shl = lshr <2 x i64> %InVec, ret <2 x i64> %shl } -; CHECK-LABEL: test_srlq_3: -; CHECK: psrlq $63, %xmm0 -; CHECK-NEXT: ret - - -; CHECK-LABEL: sra_sra_v4i32: -; CHECK: psrad $6, %xmm0 -; CHECK-NEXT: retq define <4 x i32> @sra_sra_v4i32(<4 x i32> %x) nounwind { +; CHECK-LABEL: sra_sra_v4i32: +; CHECK: # BB#0: +; CHECK-NEXT: psrad $6, %xmm0 +; CHECK-NEXT: retq %sra0 = ashr <4 x i32> %x, %sra1 = ashr <4 x i32> %sra0, ret <4 x i32> %sra1 } -; CHECK-LABEL: @srl_srl_v4i32 -; CHECK: psrld $6, %xmm0 -; CHECK-NEXT: ret define <4 x i32> @srl_srl_v4i32(<4 x i32> %x) nounwind { +; CHECK-LABEL: srl_srl_v4i32: +; CHECK: # BB#0: +; CHECK-NEXT: psrld $6, %xmm0 +; CHECK-NEXT: retq %srl0 = lshr <4 x i32> %x, %srl1 = lshr <4 x i32> %srl0, ret <4 x i32> %srl1 } -; CHECK-LABEL: @srl_shl_v4i32 -; CHECK: andps -; CHECK-NEXT: retq define <4 x i32> @srl_shl_v4i32(<4 x i32> %x) nounwind { +; CHECK-LABEL: srl_shl_v4i32: +; CHECK: # BB#0: +; CHECK-NEXT: andps {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq %srl0 = shl <4 x i32> %x, %srl1 = lshr <4 x i32> %srl0, ret <4 x i32> %srl1 } -; CHECK-LABEL: @srl_sra_31_v4i32 -; CHECK: psrld $31, %xmm0 -; CHECK-NEXT: ret define <4 x i32> @srl_sra_31_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind { +; CHECK-LABEL: srl_sra_31_v4i32: +; CHECK: # BB#0: +; CHECK-NEXT: psrld $31, %xmm0 +; CHECK-NEXT: retq %sra = ashr <4 x i32> %x, %y %srl1 = lshr <4 x i32> %sra, ret <4 x i32> %srl1 } -; CHECK-LABEL: @shl_shl_v4i32 -; CHECK: pslld $6, %xmm0 -; CHECK-NEXT: ret define <4 x i32> @shl_shl_v4i32(<4 x i32> %x) nounwind { +; CHECK-LABEL: shl_shl_v4i32: +; CHECK: # BB#0: +; CHECK-NEXT: pslld $6, %xmm0 +; CHECK-NEXT: retq %shl0 = shl <4 x i32> %x, %shl1 = shl <4 x i32> %shl0, ret <4 x i32> %shl1 } -; CHECK-LABEL: @shl_sra_v4i32 -; CHECK: andps -; CHECK-NEXT: ret define <4 x i32> @shl_sra_v4i32(<4 x i32> %x) nounwind { +; CHECK-LABEL: shl_sra_v4i32: +; CHECK: # BB#0: +; CHECK-NEXT: andps {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq %shl0 = ashr <4 x i32> %x, %shl1 = shl <4 x i32> %shl0, ret <4 x i32> %shl1 } -; CHECK-LABEL: @shl_srl_v4i32 -; CHECK: pslld $3, %xmm0 -; CHECK-NEXT: pand -; CHECK-NEXT: ret define <4 x i32> @shl_srl_v4i32(<4 x i32> %x) nounwind { +; CHECK-LABEL: shl_srl_v4i32: +; CHECK: # BB#0: +; CHECK-NEXT: pslld $3, %xmm0 +; CHECK-NEXT: pand {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq %shl0 = lshr <4 x i32> %x, %shl1 = shl <4 x i32> %shl0, ret <4 x i32> %shl1 } -; CHECK-LABEL: @shl_zext_srl_v4i32 -; CHECK: andps -; CHECK-NEXT: ret define <4 x i32> @shl_zext_srl_v4i32(<4 x i16> %x) nounwind { +; CHECK-LABEL: shl_zext_srl_v4i32: +; CHECK: # BB#0: +; CHECK-NEXT: andps {{.*}}(%rip), %xmm0 +; CHECK-NEXT: andps {{.*}}(%rip), %xmm0 +; CHECK-NEXT: retq %srl = lshr <4 x i16> %x, %zext = zext <4 x i16> %srl to <4 x i32> %shl = shl <4 x i32> %zext, ret <4 x i32> %shl } -; CHECK: @sra_trunc_srl_v4i32 -; CHECK: psrad $19, %xmm0 -; CHECK-NEXT: retq define <4 x i16> @sra_trunc_srl_v4i32(<4 x i32> %x) nounwind { +; CHECK-LABEL: sra_trunc_srl_v4i32: +; CHECK: # BB#0: +; CHECK-NEXT: psrad $19, %xmm0 +; CHECK-NEXT: retq %srl = lshr <4 x i32> %x, %trunc = trunc <4 x i32> %srl to <4 x i16> %sra = ashr <4 x i16> %trunc, ret <4 x i16> %sra } -; CHECK-LABEL: @shl_zext_shl_v4i32 -; CHECK: pand -; CHECK-NEXT: pslld $19, %xmm0 -; CHECK-NEXT: ret define <4 x i32> @shl_zext_shl_v4i32(<4 x i16> %x) nounwind { +; CHECK-LABEL: shl_zext_shl_v4i32: +; CHECK: # BB#0: +; CHECK-NEXT: pand {{.*}}(%rip), %xmm0 +; CHECK-NEXT: pslld $19, %xmm0 +; CHECK-NEXT: retq %shl0 = shl <4 x i16> %x, %ext = zext <4 x i16> %shl0 to <4 x i32> %shl1 = shl <4 x i32> %ext, ret <4 x i32> %shl1 } -; CHECK-LABEL: @sra_v4i32 -; CHECK: psrad $3, %xmm0 -; CHECK-NEXT: ret define <4 x i32> @sra_v4i32(<4 x i32> %x) nounwind { +; CHECK-LABEL: sra_v4i32: +; CHECK: # BB#0: +; CHECK-NEXT: psrad $3, %xmm0 +; CHECK-NEXT: retq %sra = ashr <4 x i32> %x, ret <4 x i32> %sra } -; CHECK-LABEL: @srl_v4i32 -; CHECK: psrld $3, %xmm0 -; CHECK-NEXT: ret define <4 x i32> @srl_v4i32(<4 x i32> %x) nounwind { +; CHECK-LABEL: srl_v4i32: +; CHECK: # BB#0: +; CHECK-NEXT: psrld $3, %xmm0 +; CHECK-NEXT: retq %sra = lshr <4 x i32> %x, ret <4 x i32> %sra } -; CHECK-LABEL: @shl_v4i32 -; CHECK: pslld $3, %xmm0 -; CHECK-NEXT: ret define <4 x i32> @shl_v4i32(<4 x i32> %x) nounwind { +; CHECK-LABEL: shl_v4i32: +; CHECK: # BB#0: +; CHECK-NEXT: pslld $3, %xmm0 +; CHECK-NEXT: retq %sra = shl <4 x i32> %x, ret <4 x i32> %sra } diff --git a/test/CodeGen/X86/sse2.ll b/test/CodeGen/X86/sse2.ll index d3ee3c6f0454..ed84905b1907 100644 --- a/test/CodeGen/X86/sse2.ll +++ b/test/CodeGen/X86/sse2.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; Tests for SSE2 and below, without SSE3+. ; RUN: llc < %s -mtriple=i386-apple-darwin10 -mcpu=pentium4 -O3 | FileCheck %s diff --git a/test/CodeGen/X86/sse3-avx-addsub-2.ll b/test/CodeGen/X86/sse3-avx-addsub-2.ll index 71efa3f8f105..79317e4576b9 100644 --- a/test/CodeGen/X86/sse3-avx-addsub-2.ll +++ b/test/CodeGen/X86/sse3-avx-addsub-2.ll @@ -1,11 +1,20 @@ -; RUN: llc < %s -march=x86-64 -mtriple=x86_64-unknown-linux-gnu -mcpu=core2 | FileCheck %s -check-prefix=CHECK -check-prefix=SSE -; RUN: llc < %s -march=x86-64 -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7-avx | FileCheck %s -check-prefix=CHECK -check-prefix=AVX - +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse3 | FileCheck %s --check-prefix=SSE +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx | FileCheck %s --check-prefix=AVX ; Verify that we correctly generate 'addsub' instructions from ; a sequence of vector extracts + float add/sub + vector inserts. define <4 x float> @test1(<4 x float> %A, <4 x float> %B) { +; SSE-LABEL: test1: +; SSE: # BB#0: +; SSE-NEXT: addsubps %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test1: +; AVX: # BB#0: +; AVX-NEXT: vaddsubps %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq %1 = extractelement <4 x float> %A, i32 0 %2 = extractelement <4 x float> %B, i32 0 %sub = fsub float %1, %2 @@ -24,13 +33,17 @@ define <4 x float> @test1(<4 x float> %A, <4 x float> %B) { %vecinsert4 = insertelement <4 x float> %vecinsert3, float %sub2, i32 2 ret <4 x float> %vecinsert4 } -; CHECK-LABEL: test1 -; SSE: addsubps -; AVX: vaddsubps -; CHECK-NEXT: ret - define <4 x float> @test2(<4 x float> %A, <4 x float> %B) { +; SSE-LABEL: test2: +; SSE: # BB#0: +; SSE-NEXT: addsubps %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test2: +; AVX: # BB#0: +; AVX-NEXT: vaddsubps %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq %1 = extractelement <4 x float> %A, i32 2 %2 = extractelement <4 x float> %B, i32 2 %sub2 = fsub float %1, %2 @@ -41,13 +54,17 @@ define <4 x float> @test2(<4 x float> %A, <4 x float> %B) { %vecinsert2 = insertelement <4 x float> %vecinsert1, float %add2, i32 3 ret <4 x float> %vecinsert2 } -; CHECK-LABEL: test2 -; SSE: addsubps -; AVX: vaddsubps -; CHECK-NEXT: ret - define <4 x float> @test3(<4 x float> %A, <4 x float> %B) { +; SSE-LABEL: test3: +; SSE: # BB#0: +; SSE-NEXT: addsubps %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test3: +; AVX: # BB#0: +; AVX-NEXT: vaddsubps %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq %1 = extractelement <4 x float> %A, i32 0 %2 = extractelement <4 x float> %B, i32 0 %sub = fsub float %1, %2 @@ -58,13 +75,17 @@ define <4 x float> @test3(<4 x float> %A, <4 x float> %B) { %vecinsert2 = insertelement <4 x float> %vecinsert1, float %add, i32 3 ret <4 x float> %vecinsert2 } -; CHECK-LABEL: test3 -; SSE: addsubps -; AVX: vaddsubps -; CHECK-NEXT: ret - define <4 x float> @test4(<4 x float> %A, <4 x float> %B) { +; SSE-LABEL: test4: +; SSE: # BB#0: +; SSE-NEXT: addsubps %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test4: +; AVX: # BB#0: +; AVX-NEXT: vaddsubps %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq %1 = extractelement <4 x float> %A, i32 2 %2 = extractelement <4 x float> %B, i32 2 %sub = fsub float %1, %2 @@ -75,13 +96,17 @@ define <4 x float> @test4(<4 x float> %A, <4 x float> %B) { %vecinsert2 = insertelement <4 x float> %vecinsert1, float %add, i32 1 ret <4 x float> %vecinsert2 } -; CHECK-LABEL: test4 -; SSE: addsubps -; AVX: vaddsubps -; CHECK-NEXT: ret - define <4 x float> @test5(<4 x float> %A, <4 x float> %B) { +; SSE-LABEL: test5: +; SSE: # BB#0: +; SSE-NEXT: addsubps %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test5: +; AVX: # BB#0: +; AVX-NEXT: vaddsubps %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq %1 = extractelement <4 x float> %A, i32 0 %2 = extractelement <4 x float> %B, i32 0 %sub2 = fsub float %1, %2 @@ -92,13 +117,17 @@ define <4 x float> @test5(<4 x float> %A, <4 x float> %B) { %vecinsert2 = insertelement <4 x float> %vecinsert1, float %add2, i32 1 ret <4 x float> %vecinsert2 } -; CHECK-LABEL: test5 -; SSE: addsubps -; AVX: vaddsubps -; CHECK-NEXT: ret - define <4 x float> @test6(<4 x float> %A, <4 x float> %B) { +; SSE-LABEL: test6: +; SSE: # BB#0: +; SSE-NEXT: addsubps %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test6: +; AVX: # BB#0: +; AVX-NEXT: vaddsubps %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq %1 = extractelement <4 x float> %A, i32 0 %2 = extractelement <4 x float> %B, i32 0 %sub = fsub float %1, %2 @@ -117,13 +146,18 @@ define <4 x float> @test6(<4 x float> %A, <4 x float> %B) { %vecinsert4 = insertelement <4 x float> %vecinsert3, float %sub2, i32 2 ret <4 x float> %vecinsert4 } -; CHECK-LABEL: test6 -; SSE: addsubps -; AVX: vaddsubps -; CHECK-NEXT: ret - define <4 x double> @test7(<4 x double> %A, <4 x double> %B) { +; SSE-LABEL: test7: +; SSE: # BB#0: +; SSE-NEXT: addsubpd %xmm2, %xmm0 +; SSE-NEXT: addsubpd %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX-LABEL: test7: +; AVX: # BB#0: +; AVX-NEXT: vaddsubpd %ymm1, %ymm0, %ymm0 +; AVX-NEXT: retq %1 = extractelement <4 x double> %A, i32 0 %2 = extractelement <4 x double> %B, i32 0 %sub = fsub double %1, %2 @@ -142,15 +176,17 @@ define <4 x double> @test7(<4 x double> %A, <4 x double> %B) { %vecinsert4 = insertelement <4 x double> %vecinsert3, double %sub2, i32 2 ret <4 x double> %vecinsert4 } -; CHECK-LABEL: test7 -; SSE: addsubpd -; SSE-NEXT: addsubpd -; AVX: vaddsubpd -; AVX-NOT: vaddsubpd -; CHECK: ret - define <2 x double> @test8(<2 x double> %A, <2 x double> %B) { +; SSE-LABEL: test8: +; SSE: # BB#0: +; SSE-NEXT: addsubpd %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test8: +; AVX: # BB#0: +; AVX-NEXT: vaddsubpd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq %1 = extractelement <2 x double> %A, i32 0 %2 = extractelement <2 x double> %B, i32 0 %sub = fsub double %1, %2 @@ -161,13 +197,18 @@ define <2 x double> @test8(<2 x double> %A, <2 x double> %B) { %vecinsert2 = insertelement <2 x double> %vecinsert1, double %add, i32 1 ret <2 x double> %vecinsert2 } -; CHECK-LABEL: test8 -; SSE: addsubpd -; AVX: vaddsubpd -; CHECK: ret - define <8 x float> @test9(<8 x float> %A, <8 x float> %B) { +; SSE-LABEL: test9: +; SSE: # BB#0: +; SSE-NEXT: addsubps %xmm2, %xmm0 +; SSE-NEXT: addsubps %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX-LABEL: test9: +; AVX: # BB#0: +; AVX-NEXT: vaddsubps %ymm1, %ymm0, %ymm0 +; AVX-NEXT: retq %1 = extractelement <8 x float> %A, i32 0 %2 = extractelement <8 x float> %B, i32 0 %sub = fsub float %1, %2 @@ -202,65 +243,118 @@ define <8 x float> @test9(<8 x float> %A, <8 x float> %B) { %vecinsert8 = insertelement <8 x float> %vecinsert7, float %sub4, i32 6 ret <8 x float> %vecinsert8 } -; CHECK-LABEL: test9 -; SSE: addsubps -; SSE-NEXT: addsubps -; AVX: vaddsubps -; AVX-NOT: vaddsubps -; CHECK: ret - ; Verify that we don't generate addsub instruction for the following ; functions. + define <4 x float> @test10(<4 x float> %A, <4 x float> %B) { +; SSE-LABEL: test10: +; SSE: # BB#0: +; SSE-NEXT: subss %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test10: +; AVX: # BB#0: +; AVX-NEXT: vsubss %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq %1 = extractelement <4 x float> %A, i32 0 %2 = extractelement <4 x float> %B, i32 0 %sub = fsub float %1, %2 %vecinsert1 = insertelement <4 x float> undef, float %sub, i32 0 ret <4 x float> %vecinsert1 } -; CHECK-LABEL: test10 -; CHECK-NOT: addsubps -; CHECK: ret - define <4 x float> @test11(<4 x float> %A, <4 x float> %B) { +; SSE-LABEL: test11: +; SSE: # BB#0: +; SSE-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] +; SSE-NEXT: shufpd {{.*#+}} xmm1 = xmm1[1,0] +; SSE-NEXT: subss %xmm1, %xmm0 +; SSE-NEXT: movddup {{.*#+}} xmm0 = xmm0[0,0] +; SSE-NEXT: retq +; +; AVX-LABEL: test11: +; AVX: # BB#0: +; AVX-NEXT: vpermilpd {{.*#+}} xmm0 = xmm0[1,0] +; AVX-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0] +; AVX-NEXT: vsubss %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0] +; AVX-NEXT: retq %1 = extractelement <4 x float> %A, i32 2 %2 = extractelement <4 x float> %B, i32 2 %sub = fsub float %1, %2 %vecinsert1 = insertelement <4 x float> undef, float %sub, i32 2 ret <4 x float> %vecinsert1 } -; CHECK-LABEL: test11 -; CHECK-NOT: addsubps -; CHECK: ret - define <4 x float> @test12(<4 x float> %A, <4 x float> %B) { +; SSE-LABEL: test12: +; SSE: # BB#0: +; SSE-NEXT: movshdup {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE-NEXT: movshdup {{.*#+}} xmm1 = xmm1[1,1,3,3] +; SSE-NEXT: addss %xmm0, %xmm1 +; SSE-NEXT: movsldup {{.*#+}} xmm0 = xmm1[0,0,2,2] +; SSE-NEXT: retq +; +; AVX-LABEL: test12: +; AVX: # BB#0: +; AVX-NEXT: vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3] +; AVX-NEXT: vmovshdup {{.*#+}} xmm1 = xmm1[1,1,3,3] +; AVX-NEXT: vaddss %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmovsldup {{.*#+}} xmm0 = xmm0[0,0,2,2] +; AVX-NEXT: retq %1 = extractelement <4 x float> %A, i32 1 %2 = extractelement <4 x float> %B, i32 1 %add = fadd float %1, %2 %vecinsert1 = insertelement <4 x float> undef, float %add, i32 1 ret <4 x float> %vecinsert1 } -; CHECK-LABEL: test12 -; CHECK-NOT: addsubps -; CHECK: ret - define <4 x float> @test13(<4 x float> %A, <4 x float> %B) { +; SSE-LABEL: test13: +; SSE: # BB#0: +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,1,2,3] +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1,2,3] +; SSE-NEXT: addss %xmm0, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1,2,0] +; SSE-NEXT: movaps %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test13: +; AVX: # BB#0: +; AVX-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3] +; AVX-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3] +; AVX-NEXT: vaddss %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[0,1,2,0] +; AVX-NEXT: retq %1 = extractelement <4 x float> %A, i32 3 %2 = extractelement <4 x float> %B, i32 3 %add = fadd float %1, %2 %vecinsert1 = insertelement <4 x float> undef, float %add, i32 3 ret <4 x float> %vecinsert1 } -; CHECK-LABEL: test13 -; CHECK-NOT: addsubps -; CHECK: ret - define <4 x float> @test14(<4 x float> %A, <4 x float> %B) { +; SSE-LABEL: test14: +; SSE: # BB#0: +; SSE-NEXT: movaps %xmm0, %xmm2 +; SSE-NEXT: subss %xmm1, %xmm2 +; SSE-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] +; SSE-NEXT: shufpd {{.*#+}} xmm1 = xmm1[1,0] +; SSE-NEXT: subss %xmm1, %xmm0 +; SSE-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1,1,3] +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test14: +; AVX: # BB#0: +; AVX-NEXT: vsubss %xmm1, %xmm0, %xmm2 +; AVX-NEXT: vpermilpd {{.*#+}} xmm0 = xmm0[1,0] +; AVX-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0] +; AVX-NEXT: vsubss %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm2[0,1],xmm0[0],xmm2[3] +; AVX-NEXT: retq %1 = extractelement <4 x float> %A, i32 0 %2 = extractelement <4 x float> %B, i32 0 %sub = fsub float %1, %2 @@ -271,12 +365,32 @@ define <4 x float> @test14(<4 x float> %A, <4 x float> %B) { %vecinsert2 = insertelement <4 x float> %vecinsert1, float %sub2, i32 2 ret <4 x float> %vecinsert2 } -; CHECK-LABEL: test14 -; CHECK-NOT: addsubps -; CHECK: ret - define <4 x float> @test15(<4 x float> %A, <4 x float> %B) { +; SSE-LABEL: test15: +; SSE: # BB#0: +; SSE-NEXT: movshdup {{.*#+}} xmm3 = xmm0[1,1,3,3] +; SSE-NEXT: movshdup {{.*#+}} xmm2 = xmm1[1,1,3,3] +; SSE-NEXT: addss %xmm3, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,1,2,3] +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1,2,3] +; SSE-NEXT: addss %xmm0, %xmm1 +; SSE-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,0,2,1] +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test15: +; AVX: # BB#0: +; AVX-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3] +; AVX-NEXT: vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3] +; AVX-NEXT: vaddss %xmm3, %xmm2, %xmm2 +; AVX-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3] +; AVX-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3] +; AVX-NEXT: vaddss %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[0],xmm2[0],xmm0[2,3] +; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0] +; AVX-NEXT: retq %1 = extractelement <4 x float> %A, i32 1 %2 = extractelement <4 x float> %B, i32 1 %add = fadd float %1, %2 @@ -287,12 +401,43 @@ define <4 x float> @test15(<4 x float> %A, <4 x float> %B) { %vecinsert2 = insertelement <4 x float> %vecinsert1, float %add2, i32 3 ret <4 x float> %vecinsert2 } -; CHECK-LABEL: test15 -; CHECK-NOT: addsubps -; CHECK: ret - define <4 x float> @test16(<4 x float> %A, <4 x float> %B) { +; SSE-LABEL: test16: +; SSE: # BB#0: +; SSE-NEXT: movaps %xmm0, %xmm2 +; SSE-NEXT: subss %xmm0, %xmm2 +; SSE-NEXT: movaps %xmm0, %xmm3 +; SSE-NEXT: shufpd {{.*#+}} xmm3 = xmm3[1,0] +; SSE-NEXT: movapd %xmm1, %xmm4 +; SSE-NEXT: shufpd {{.*#+}} xmm4 = xmm4[1,0] +; SSE-NEXT: subss %xmm4, %xmm3 +; SSE-NEXT: movshdup {{.*#+}} xmm4 = xmm0[1,1,3,3] +; SSE-NEXT: addss %xmm0, %xmm4 +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,1,2,3] +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1,2,3] +; SSE-NEXT: addss %xmm0, %xmm1 +; SSE-NEXT: unpcklps {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1] +; SSE-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] +; SSE-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1] +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test16: +; AVX: # BB#0: +; AVX-NEXT: vsubss %xmm0, %xmm0, %xmm2 +; AVX-NEXT: vpermilpd {{.*#+}} xmm3 = xmm0[1,0] +; AVX-NEXT: vpermilpd {{.*#+}} xmm4 = xmm1[1,0] +; AVX-NEXT: vsubss %xmm4, %xmm3, %xmm3 +; AVX-NEXT: vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3] +; AVX-NEXT: vaddss %xmm0, %xmm4, %xmm4 +; AVX-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3] +; AVX-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3] +; AVX-NEXT: vaddss %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vinsertps {{.*#+}} xmm1 = xmm2[0],xmm4[0],xmm2[2,3] +; AVX-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0],xmm1[3] +; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0] +; AVX-NEXT: retq %1 = extractelement <4 x float> %A, i32 0 %2 = extractelement <4 x float> %B, i32 0 %sub = fsub float %1, undef @@ -311,11 +456,17 @@ define <4 x float> @test16(<4 x float> %A, <4 x float> %B) { %vecinsert4 = insertelement <4 x float> %vecinsert3, float %sub2, i32 2 ret <4 x float> %vecinsert4 } -; CHECK-LABEL: test16 -; CHECK-NOT: addsubps -; CHECK: ret define <2 x float> @test_v2f32(<2 x float> %v0, <2 x float> %v1) { +; SSE-LABEL: test_v2f32: +; SSE: # BB#0: +; SSE-NEXT: addsubps %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test_v2f32: +; AVX: # BB#0: +; AVX-NEXT: vaddsubps %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq %v2 = extractelement <2 x float> %v0, i32 0 %v3 = extractelement <2 x float> %v1, i32 0 %v4 = extractelement <2 x float> %v0, i32 1 @@ -326,6 +477,3 @@ define <2 x float> @test_v2f32(<2 x float> %v0, <2 x float> %v1) { %res1 = insertelement <2 x float> %res0, float %add, i32 1 ret <2 x float> %res1 } -; CHECK-LABEL: test_v2f32 -; CHECK: addsubps %xmm1, %xmm0 -; CHECK-NEXT: retq diff --git a/test/CodeGen/X86/sse3-avx-addsub.ll b/test/CodeGen/X86/sse3-avx-addsub.ll index 76141fc876ae..8665edf8f1d5 100644 --- a/test/CodeGen/X86/sse3-avx-addsub.ll +++ b/test/CodeGen/X86/sse3-avx-addsub.ll @@ -1,5 +1,6 @@ -; RUN: llc < %s -march=x86-64 -mcpu=core2 | FileCheck %s -check-prefix=SSE -check-prefix=CHECK -; RUN: llc < %s -march=x86-64 -mcpu=corei7-avx | FileCheck %s -check-prefix=AVX -check-prefix=CHECK +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse3 | FileCheck %s --check-prefix=SSE +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx | FileCheck %s --check-prefix=AVX ; Test ADDSUB ISel patterns. @@ -35,109 +36,207 @@ ; } define <4 x float> @test1(<4 x float> %A, <4 x float> %B) { +; SSE-LABEL: test1: +; SSE: # BB#0: +; SSE-NEXT: addsubps %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test1: +; AVX: # BB#0: +; AVX-NEXT: vaddsubps %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq %sub = fsub <4 x float> %A, %B %add = fadd <4 x float> %A, %B %vecinit6 = shufflevector <4 x float> %sub, <4 x float> %add, <4 x i32> ret <4 x float> %vecinit6 } -; CHECK-LABEL: test1 -; SSE: addsubps -; AVX: vaddsubps -; CHECK-NEXT: ret - define <8 x float> @test2(<8 x float> %A, <8 x float> %B) { +; SSE-LABEL: test2: +; SSE: # BB#0: +; SSE-NEXT: addsubps %xmm2, %xmm0 +; SSE-NEXT: addsubps %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX-LABEL: test2: +; AVX: # BB#0: +; AVX-NEXT: vaddsubps %ymm1, %ymm0, %ymm0 +; AVX-NEXT: retq %sub = fsub <8 x float> %A, %B %add = fadd <8 x float> %A, %B %vecinit14 = shufflevector <8 x float> %sub, <8 x float> %add, <8 x i32> ret <8 x float> %vecinit14 } -; CHECK-LABEL: test2 -; SSE: addsubps -; SSE-NEXT: addsubps -; AVX: vaddsubps -; AVX-NOT: vaddsubps -; CHECK: ret - define <4 x double> @test3(<4 x double> %A, <4 x double> %B) { +; SSE-LABEL: test3: +; SSE: # BB#0: +; SSE-NEXT: addsubpd %xmm2, %xmm0 +; SSE-NEXT: addsubpd %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX-LABEL: test3: +; AVX: # BB#0: +; AVX-NEXT: vaddsubpd %ymm1, %ymm0, %ymm0 +; AVX-NEXT: retq %sub = fsub <4 x double> %A, %B %add = fadd <4 x double> %A, %B %vecinit6 = shufflevector <4 x double> %sub, <4 x double> %add, <4 x i32> ret <4 x double> %vecinit6 } -; CHECK-LABEL: test3 -; SSE: addsubpd -; SSE: addsubpd -; AVX: vaddsubpd -; AVX-NOT: vaddsubpd -; CHECK: ret - define <2 x double> @test4(<2 x double> %A, <2 x double> %B) #0 { +; SSE-LABEL: test4: +; SSE: # BB#0: +; SSE-NEXT: addsubpd %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test4: +; AVX: # BB#0: +; AVX-NEXT: vaddsubpd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq %add = fadd <2 x double> %A, %B %sub = fsub <2 x double> %A, %B %vecinit2 = shufflevector <2 x double> %sub, <2 x double> %add, <2 x i32> ret <2 x double> %vecinit2 } -; CHECK-LABEL: test4 -; SSE: addsubpd -; AVX: vaddsubpd -; CHECK-NEXT: ret - define <4 x float> @test1b(<4 x float> %A, <4 x float>* %B) { +; SSE-LABEL: test1b: +; SSE: # BB#0: +; SSE-NEXT: addsubps (%rdi), %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test1b: +; AVX: # BB#0: +; AVX-NEXT: vaddsubps (%rdi), %xmm0, %xmm0 +; AVX-NEXT: retq %1 = load <4 x float>, <4 x float>* %B %add = fadd <4 x float> %A, %1 %sub = fsub <4 x float> %A, %1 %vecinit6 = shufflevector <4 x float> %sub, <4 x float> %add, <4 x i32> ret <4 x float> %vecinit6 } -; CHECK-LABEL: test1b -; SSE: addsubps -; AVX: vaddsubps -; CHECK-NEXT: ret - define <8 x float> @test2b(<8 x float> %A, <8 x float>* %B) { +; SSE-LABEL: test2b: +; SSE: # BB#0: +; SSE-NEXT: addsubps (%rdi), %xmm0 +; SSE-NEXT: addsubps 16(%rdi), %xmm1 +; SSE-NEXT: retq +; +; AVX-LABEL: test2b: +; AVX: # BB#0: +; AVX-NEXT: vaddsubps (%rdi), %ymm0, %ymm0 +; AVX-NEXT: retq %1 = load <8 x float>, <8 x float>* %B %add = fadd <8 x float> %A, %1 %sub = fsub <8 x float> %A, %1 %vecinit14 = shufflevector <8 x float> %sub, <8 x float> %add, <8 x i32> ret <8 x float> %vecinit14 } -; CHECK-LABEL: test2b -; SSE: addsubps -; SSE-NEXT: addsubps -; AVX: vaddsubps -; AVX-NOT: vaddsubps -; CHECK: ret - define <4 x double> @test3b(<4 x double> %A, <4 x double>* %B) { +; SSE-LABEL: test3b: +; SSE: # BB#0: +; SSE-NEXT: addsubpd (%rdi), %xmm0 +; SSE-NEXT: addsubpd 16(%rdi), %xmm1 +; SSE-NEXT: retq +; +; AVX-LABEL: test3b: +; AVX: # BB#0: +; AVX-NEXT: vaddsubpd (%rdi), %ymm0, %ymm0 +; AVX-NEXT: retq %1 = load <4 x double>, <4 x double>* %B %add = fadd <4 x double> %A, %1 %sub = fsub <4 x double> %A, %1 %vecinit6 = shufflevector <4 x double> %sub, <4 x double> %add, <4 x i32> ret <4 x double> %vecinit6 } -; CHECK-LABEL: test3b -; SSE: addsubpd -; SSE: addsubpd -; AVX: vaddsubpd -; AVX-NOT: vaddsubpd -; CHECK: ret - define <2 x double> @test4b(<2 x double> %A, <2 x double>* %B) { +; SSE-LABEL: test4b: +; SSE: # BB#0: +; SSE-NEXT: addsubpd (%rdi), %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test4b: +; AVX: # BB#0: +; AVX-NEXT: vaddsubpd (%rdi), %xmm0, %xmm0 +; AVX-NEXT: retq %1 = load <2 x double>, <2 x double>* %B %sub = fsub <2 x double> %A, %1 %add = fadd <2 x double> %A, %1 %vecinit2 = shufflevector <2 x double> %sub, <2 x double> %add, <2 x i32> ret <2 x double> %vecinit2 } -; CHECK-LABEL: test4b -; SSE: addsubpd -; AVX: vaddsubpd -; CHECK-NEXT: ret +define <4 x float> @test1c(<4 x float> %A, <4 x float>* %B) { +; SSE-LABEL: test1c: +; SSE: # BB#0: +; SSE-NEXT: addsubps (%rdi), %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test1c: +; AVX: # BB#0: +; AVX-NEXT: vaddsubps (%rdi), %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = load <4 x float>, <4 x float>* %B + %add = fadd <4 x float> %A, %1 + %sub = fsub <4 x float> %A, %1 + %vecinit6 = shufflevector <4 x float> %add, <4 x float> %sub, <4 x i32> + ret <4 x float> %vecinit6 +} + +define <8 x float> @test2c(<8 x float> %A, <8 x float>* %B) { +; SSE-LABEL: test2c: +; SSE: # BB#0: +; SSE-NEXT: addsubps (%rdi), %xmm0 +; SSE-NEXT: addsubps 16(%rdi), %xmm1 +; SSE-NEXT: retq +; +; AVX-LABEL: test2c: +; AVX: # BB#0: +; AVX-NEXT: vaddsubps (%rdi), %ymm0, %ymm0 +; AVX-NEXT: retq + %1 = load <8 x float>, <8 x float>* %B + %add = fadd <8 x float> %A, %1 + %sub = fsub <8 x float> %A, %1 + %vecinit14 = shufflevector <8 x float> %add, <8 x float> %sub, <8 x i32> + ret <8 x float> %vecinit14 +} + +define <4 x double> @test3c(<4 x double> %A, <4 x double>* %B) { +; SSE-LABEL: test3c: +; SSE: # BB#0: +; SSE-NEXT: addsubpd (%rdi), %xmm0 +; SSE-NEXT: addsubpd 16(%rdi), %xmm1 +; SSE-NEXT: retq +; +; AVX-LABEL: test3c: +; AVX: # BB#0: +; AVX-NEXT: vaddsubpd (%rdi), %ymm0, %ymm0 +; AVX-NEXT: retq + %1 = load <4 x double>, <4 x double>* %B + %add = fadd <4 x double> %A, %1 + %sub = fsub <4 x double> %A, %1 + %vecinit6 = shufflevector <4 x double> %add, <4 x double> %sub, <4 x i32> + ret <4 x double> %vecinit6 +} + +define <2 x double> @test4c(<2 x double> %A, <2 x double>* %B) { +; SSE-LABEL: test4c: +; SSE: # BB#0: +; SSE-NEXT: addsubpd (%rdi), %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test4c: +; AVX: # BB#0: +; AVX-NEXT: vaddsubpd (%rdi), %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = load <2 x double>, <2 x double>* %B + %sub = fsub <2 x double> %A, %1 + %add = fadd <2 x double> %A, %1 + %vecinit2 = shufflevector <2 x double> %add, <2 x double> %sub, <2 x i32> + ret <2 x double> %vecinit2 +} diff --git a/test/CodeGen/X86/sse3-intrinsics-fast-isel.ll b/test/CodeGen/X86/sse3-intrinsics-fast-isel.ll new file mode 100644 index 000000000000..217be9aeae3a --- /dev/null +++ b/test/CodeGen/X86/sse3-intrinsics-fast-isel.ll @@ -0,0 +1,171 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -fast-isel -mtriple=i386-unknown-unknown -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=X32 +; RUN: llc < %s -fast-isel -mtriple=x86_64-unknown-unknown -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=X64 + +; NOTE: This should use IR equivalent to what is generated by clang/test/CodeGen/sse3-builtins.c + +define <2 x double> @test_mm_addsub_pd(<2 x double> %a0, <2 x double> %a1) { +; X32-LABEL: test_mm_addsub_pd: +; X32: # BB#0: +; X32-NEXT: addsubpd %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_addsub_pd: +; X64: # BB#0: +; X64-NEXT: addsubpd %xmm1, %xmm0 +; X64-NEXT: retq + %res = call <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> %a0, <2 x double> %a1) + ret <2 x double> %res +} +declare <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double>, <2 x double>) nounwind readnone + +define <4 x float> @test_mm_addsub_ps(<4 x float> %a0, <4 x float> %a1) { +; X32-LABEL: test_mm_addsub_ps: +; X32: # BB#0: +; X32-NEXT: addsubps %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_addsub_ps: +; X64: # BB#0: +; X64-NEXT: addsubps %xmm1, %xmm0 +; X64-NEXT: retq + %res = call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> %a0, <4 x float> %a1) + ret <4 x float> %res +} +declare <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float>, <4 x float>) nounwind readnone + +define <2 x double> @test_mm_hadd_pd(<2 x double> %a0, <2 x double> %a1) { +; X32-LABEL: test_mm_hadd_pd: +; X32: # BB#0: +; X32-NEXT: haddpd %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_hadd_pd: +; X64: # BB#0: +; X64-NEXT: haddpd %xmm1, %xmm0 +; X64-NEXT: retq + %res = call <2 x double> @llvm.x86.sse3.hadd.pd(<2 x double> %a0, <2 x double> %a1) + ret <2 x double> %res +} +declare <2 x double> @llvm.x86.sse3.hadd.pd(<2 x double>, <2 x double>) nounwind readnone + +define <4 x float> @test_mm_hadd_ps(<4 x float> %a0, <4 x float> %a1) { +; X32-LABEL: test_mm_hadd_ps: +; X32: # BB#0: +; X32-NEXT: haddps %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_hadd_ps: +; X64: # BB#0: +; X64-NEXT: haddps %xmm1, %xmm0 +; X64-NEXT: retq + %res = call <4 x float> @llvm.x86.sse3.hadd.ps(<4 x float> %a0, <4 x float> %a1) + ret <4 x float> %res +} +declare <4 x float> @llvm.x86.sse3.hadd.ps(<4 x float>, <4 x float>) nounwind readnone + +define <2 x double> @test_mm_hsub_pd(<2 x double> %a0, <2 x double> %a1) { +; X32-LABEL: test_mm_hsub_pd: +; X32: # BB#0: +; X32-NEXT: hsubpd %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_hsub_pd: +; X64: # BB#0: +; X64-NEXT: hsubpd %xmm1, %xmm0 +; X64-NEXT: retq + %res = call <2 x double> @llvm.x86.sse3.hsub.pd(<2 x double> %a0, <2 x double> %a1) + ret <2 x double> %res +} +declare <2 x double> @llvm.x86.sse3.hsub.pd(<2 x double>, <2 x double>) nounwind readnone + +define <4 x float> @test_mm_hsub_ps(<4 x float> %a0, <4 x float> %a1) { +; X32-LABEL: test_mm_hsub_ps: +; X32: # BB#0: +; X32-NEXT: hsubps %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_hsub_ps: +; X64: # BB#0: +; X64-NEXT: hsubps %xmm1, %xmm0 +; X64-NEXT: retq + %res = call <4 x float> @llvm.x86.sse3.hsub.ps(<4 x float> %a0, <4 x float> %a1) + ret <4 x float> %res +} +declare <4 x float> @llvm.x86.sse3.hsub.ps(<4 x float>, <4 x float>) nounwind readnone + +define <2 x i64> @test_mm_lddqu_si128(i8* %a0) { +; X32-LABEL: test_mm_lddqu_si128: +; X32: # BB#0: +; X32-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-NEXT: lddqu (%eax), %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_lddqu_si128: +; X64: # BB#0: +; X64-NEXT: lddqu (%rdi), %xmm0 +; X64-NEXT: retq + %call = call <16 x i8> @llvm.x86.sse3.ldu.dq(i8* %a0) + %res = bitcast <16 x i8> %call to <2 x i64> + ret <2 x i64> %res +} +declare <16 x i8> @llvm.x86.sse3.ldu.dq(i8*) nounwind readonly + +define <2 x double> @test_mm_loaddup_pd(double* %a0) { +; X32-LABEL: test_mm_loaddup_pd: +; X32: # BB#0: +; X32-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-NEXT: movddup (%eax), %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_loaddup_pd: +; X64: # BB#0: +; X64-NEXT: movddup (%rdi), %xmm0 +; X64-NEXT: retq + %ld = load double, double* %a0 + %res0 = insertelement <2 x double> undef, double %ld, i32 0 + %res1 = insertelement <2 x double> %res0, double %ld, i32 1 + ret <2 x double> %res1 +} + +define <2 x double> @test_mm_movedup_pd(<2 x double> %a0) { +; X32-LABEL: test_mm_movedup_pd: +; X32: # BB#0: +; X32-NEXT: movddup {{.*#+}} xmm0 = xmm0[0,0] +; X32-NEXT: retl +; +; X64-LABEL: test_mm_movedup_pd: +; X64: # BB#0: +; X64-NEXT: movddup {{.*#+}} xmm0 = xmm0[0,0] +; X64-NEXT: retq + %res = shufflevector <2 x double> %a0, <2 x double> %a0, <2 x i32> zeroinitializer + ret <2 x double> %res +} + +define <4 x float> @test_mm_movehdup_ps(<4 x float> %a0) { +; X32-LABEL: test_mm_movehdup_ps: +; X32: # BB#0: +; X32-NEXT: movshdup {{.*#+}} xmm0 = xmm0[1,1,3,3] +; X32-NEXT: retl +; +; X64-LABEL: test_mm_movehdup_ps: +; X64: # BB#0: +; X64-NEXT: movshdup {{.*#+}} xmm0 = xmm0[1,1,3,3] +; X64-NEXT: retq + %res = shufflevector <4 x float> %a0, <4 x float> %a0, <4 x i32> + ret <4 x float> %res +} + +define <4 x float> @test_mm_moveldup_ps(<4 x float> %a0) { +; X32-LABEL: test_mm_moveldup_ps: +; X32: # BB#0: +; X32-NEXT: movsldup {{.*#+}} xmm0 = xmm0[0,0,2,2] +; X32-NEXT: retl +; +; X64-LABEL: test_mm_moveldup_ps: +; X64: # BB#0: +; X64-NEXT: movsldup {{.*#+}} xmm0 = xmm0[0,0,2,2] +; X64-NEXT: retq + %res = shufflevector <4 x float> %a0, <4 x float> %a0, <4 x i32> + ret <4 x float> %res +} diff --git a/test/CodeGen/X86/sse3.ll b/test/CodeGen/X86/sse3.ll index 398675276c66..2c24478706e6 100644 --- a/test/CodeGen/X86/sse3.ll +++ b/test/CodeGen/X86/sse3.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; These are tests for SSE3 codegen. ; RUN: llc < %s -mtriple=x86_64-apple-darwin9 --mattr=+sse3 | FileCheck %s --check-prefix=X64 @@ -269,8 +270,10 @@ entry: define <4 x i32> @t17() nounwind { ; X64-LABEL: t17: ; X64: ## BB#0: ## %entry -; X64-NEXT: movddup {{.*#+}} xmm0 = mem[0,0] -; X64-NEXT: andpd {{.*}}(%rip), %xmm0 +; X64-NEXT: movaps (%rax), %xmm0 +; X64-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0,0,1,1] +; X64-NEXT: pxor %xmm1, %xmm1 +; X64-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; X64-NEXT: retq entry: %tmp1 = load <4 x float>, <4 x float>* undef, align 16 diff --git a/test/CodeGen/X86/sse41-intrinsics-x86-upgrade.ll b/test/CodeGen/X86/sse41-intrinsics-x86-upgrade.ll index 6fab98e70a89..75f69ffd6db9 100644 --- a/test/CodeGen/X86/sse41-intrinsics-x86-upgrade.ll +++ b/test/CodeGen/X86/sse41-intrinsics-x86-upgrade.ll @@ -42,7 +42,6 @@ define <4 x float> @test_x86_sse41_insertps(<4 x float> %a0, <4 x float> %a1) { declare <4 x float> @llvm.x86.sse41.insertps(<4 x float>, <4 x float>, i32) nounwind readnone - define <8 x i16> @test_x86_sse41_mpsadbw(<16 x i8> %a0, <16 x i8> %a1) { ; CHECK: mpsadbw %res = call <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8> %a0, <16 x i8> %a1, i32 7) ; <<8 x i16>> [#uses=1] @@ -59,3 +58,49 @@ define <8 x i16> @test_x86_sse41_pblendw(<8 x i16> %a0, <8 x i16> %a1) { declare <8 x i16> @llvm.x86.sse41.pblendw(<8 x i16>, <8 x i16>, i32) nounwind readnone +define <4 x i32> @test_x86_sse41_pmovsxbd(<16 x i8> %a0) { + ; CHECK: pmovsxbd + %res = call <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8> %a0) ; <<4 x i32>> [#uses=1] + ret <4 x i32> %res +} +declare <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8>) nounwind readnone + + +define <2 x i64> @test_x86_sse41_pmovsxbq(<16 x i8> %a0) { + ; CHECK: pmovsxbq + %res = call <2 x i64> @llvm.x86.sse41.pmovsxbq(<16 x i8> %a0) ; <<2 x i64>> [#uses=1] + ret <2 x i64> %res +} +declare <2 x i64> @llvm.x86.sse41.pmovsxbq(<16 x i8>) nounwind readnone + + +define <8 x i16> @test_x86_sse41_pmovsxbw(<16 x i8> %a0) { + ; CHECK: pmovsxbw + %res = call <8 x i16> @llvm.x86.sse41.pmovsxbw(<16 x i8> %a0) ; <<8 x i16>> [#uses=1] + ret <8 x i16> %res +} +declare <8 x i16> @llvm.x86.sse41.pmovsxbw(<16 x i8>) nounwind readnone + + +define <2 x i64> @test_x86_sse41_pmovsxdq(<4 x i32> %a0) { + ; CHECK: pmovsxdq + %res = call <2 x i64> @llvm.x86.sse41.pmovsxdq(<4 x i32> %a0) ; <<2 x i64>> [#uses=1] + ret <2 x i64> %res +} +declare <2 x i64> @llvm.x86.sse41.pmovsxdq(<4 x i32>) nounwind readnone + + +define <4 x i32> @test_x86_sse41_pmovsxwd(<8 x i16> %a0) { + ; CHECK: pmovsxwd + %res = call <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16> %a0) ; <<4 x i32>> [#uses=1] + ret <4 x i32> %res +} +declare <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16>) nounwind readnone + + +define <2 x i64> @test_x86_sse41_pmovsxwq(<8 x i16> %a0) { + ; CHECK: pmovsxwq + %res = call <2 x i64> @llvm.x86.sse41.pmovsxwq(<8 x i16> %a0) ; <<2 x i64>> [#uses=1] + ret <2 x i64> %res +} +declare <2 x i64> @llvm.x86.sse41.pmovsxwq(<8 x i16>) nounwind readnone diff --git a/test/CodeGen/X86/sse41-intrinsics-x86.ll b/test/CodeGen/X86/sse41-intrinsics-x86.ll index 771e4024336c..ceff4f9782e9 100644 --- a/test/CodeGen/X86/sse41-intrinsics-x86.ll +++ b/test/CodeGen/X86/sse41-intrinsics-x86.ll @@ -162,54 +162,6 @@ define <8 x i16> @test_x86_sse41_pminuw(<8 x i16> %a0, <8 x i16> %a1) { declare <8 x i16> @llvm.x86.sse41.pminuw(<8 x i16>, <8 x i16>) nounwind readnone -define <4 x i32> @test_x86_sse41_pmovsxbd(<16 x i8> %a0) { - ; CHECK: pmovsxbd - %res = call <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8> %a0) ; <<4 x i32>> [#uses=1] - ret <4 x i32> %res -} -declare <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8>) nounwind readnone - - -define <2 x i64> @test_x86_sse41_pmovsxbq(<16 x i8> %a0) { - ; CHECK: pmovsxbq - %res = call <2 x i64> @llvm.x86.sse41.pmovsxbq(<16 x i8> %a0) ; <<2 x i64>> [#uses=1] - ret <2 x i64> %res -} -declare <2 x i64> @llvm.x86.sse41.pmovsxbq(<16 x i8>) nounwind readnone - - -define <8 x i16> @test_x86_sse41_pmovsxbw(<16 x i8> %a0) { - ; CHECK: pmovsxbw - %res = call <8 x i16> @llvm.x86.sse41.pmovsxbw(<16 x i8> %a0) ; <<8 x i16>> [#uses=1] - ret <8 x i16> %res -} -declare <8 x i16> @llvm.x86.sse41.pmovsxbw(<16 x i8>) nounwind readnone - - -define <2 x i64> @test_x86_sse41_pmovsxdq(<4 x i32> %a0) { - ; CHECK: pmovsxdq - %res = call <2 x i64> @llvm.x86.sse41.pmovsxdq(<4 x i32> %a0) ; <<2 x i64>> [#uses=1] - ret <2 x i64> %res -} -declare <2 x i64> @llvm.x86.sse41.pmovsxdq(<4 x i32>) nounwind readnone - - -define <4 x i32> @test_x86_sse41_pmovsxwd(<8 x i16> %a0) { - ; CHECK: pmovsxwd - %res = call <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16> %a0) ; <<4 x i32>> [#uses=1] - ret <4 x i32> %res -} -declare <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16>) nounwind readnone - - -define <2 x i64> @test_x86_sse41_pmovsxwq(<8 x i16> %a0) { - ; CHECK: pmovsxwq - %res = call <2 x i64> @llvm.x86.sse41.pmovsxwq(<8 x i16> %a0) ; <<2 x i64>> [#uses=1] - ret <2 x i64> %res -} -declare <2 x i64> @llvm.x86.sse41.pmovsxwq(<8 x i16>) nounwind readnone - - define <4 x i32> @test_x86_sse41_pmovzxbd(<16 x i8> %a0) { ; CHECK: pmovzxbd %res = call <4 x i32> @llvm.x86.sse41.pmovzxbd(<16 x i8> %a0) ; <<4 x i32>> [#uses=1] diff --git a/test/CodeGen/X86/sse41-pmovxrm-intrinsics.ll b/test/CodeGen/X86/sse41-pmovxrm-intrinsics.ll index a16e79277143..a7e48d8ac038 100644 --- a/test/CodeGen/X86/sse41-pmovxrm-intrinsics.ll +++ b/test/CodeGen/X86/sse41-pmovxrm-intrinsics.ll @@ -1,109 +1,188 @@ -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE41 -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=AVX +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+sse4.1 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE41 +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=AVX define <8 x i16> @test_llvm_x86_sse41_pmovsxbw(<16 x i8>* %a) { -; CHECK-LABEL: test_llvm_x86_sse41_pmovsxbw -; SSE41: pmovsxbw (%rdi), %xmm0 -; AVX: vpmovsxbw (%rdi), %xmm0 +; SSE41-LABEL: test_llvm_x86_sse41_pmovsxbw: +; SSE41: ## BB#0: +; SSE41-NEXT: pmovsxbw (%rdi), %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: test_llvm_x86_sse41_pmovsxbw: +; AVX: ## BB#0: +; AVX-NEXT: vpmovsxbw (%rdi), %xmm0 +; AVX-NEXT: retq %1 = load <16 x i8>, <16 x i8>* %a, align 1 - %2 = call <8 x i16> @llvm.x86.sse41.pmovsxbw(<16 x i8> %1) - ret <8 x i16> %2 + %2 = shufflevector <16 x i8> %1, <16 x i8> undef, <8 x i32> + %3 = sext <8 x i8> %2 to <8 x i16> + ret <8 x i16> %3 } define <4 x i32> @test_llvm_x86_sse41_pmovsxbd(<16 x i8>* %a) { -; CHECK-LABEL: test_llvm_x86_sse41_pmovsxbd -; SSE41: pmovsxbd (%rdi), %xmm0 -; AVX: vpmovsxbd (%rdi), %xmm0 +; SSE41-LABEL: test_llvm_x86_sse41_pmovsxbd: +; SSE41: ## BB#0: +; SSE41-NEXT: pmovsxbd (%rdi), %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: test_llvm_x86_sse41_pmovsxbd: +; AVX: ## BB#0: +; AVX-NEXT: vpmovsxbd (%rdi), %xmm0 +; AVX-NEXT: retq %1 = load <16 x i8>, <16 x i8>* %a, align 1 - %2 = call <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8> %1) - ret <4 x i32> %2 + %2 = shufflevector <16 x i8> %1, <16 x i8> undef, <4 x i32> + %3 = sext <4 x i8> %2 to <4 x i32> + ret <4 x i32> %3 } define <2 x i64> @test_llvm_x86_sse41_pmovsxbq(<16 x i8>* %a) { -; CHECK-LABEL: test_llvm_x86_sse41_pmovsxbq -; SSE41: pmovsxbq (%rdi), %xmm0 -; AVX: vpmovsxbq (%rdi), %xmm0 +; SSE41-LABEL: test_llvm_x86_sse41_pmovsxbq: +; SSE41: ## BB#0: +; SSE41-NEXT: pmovsxbq (%rdi), %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: test_llvm_x86_sse41_pmovsxbq: +; AVX: ## BB#0: +; AVX-NEXT: vpmovsxbq (%rdi), %xmm0 +; AVX-NEXT: retq %1 = load <16 x i8>, <16 x i8>* %a, align 1 - %2 = call <2 x i64> @llvm.x86.sse41.pmovsxbq(<16 x i8> %1) - ret <2 x i64> %2 + %2 = shufflevector <16 x i8> %1, <16 x i8> undef, <2 x i32> + %3 = sext <2 x i8> %2 to <2 x i64> + ret <2 x i64> %3 } define <4 x i32> @test_llvm_x86_sse41_pmovsxwd(<8 x i16>* %a) { -; CHECK-LABEL: test_llvm_x86_sse41_pmovsxwd -; SSE41: pmovsxwd (%rdi), %xmm0 -; AVX: vpmovsxwd (%rdi), %xmm0 +; SSE41-LABEL: test_llvm_x86_sse41_pmovsxwd: +; SSE41: ## BB#0: +; SSE41-NEXT: pmovsxwd (%rdi), %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: test_llvm_x86_sse41_pmovsxwd: +; AVX: ## BB#0: +; AVX-NEXT: vpmovsxwd (%rdi), %xmm0 +; AVX-NEXT: retq %1 = load <8 x i16>, <8 x i16>* %a, align 1 - %2 = call <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16> %1) - ret <4 x i32> %2 + %2 = shufflevector <8 x i16> %1, <8 x i16> undef, <4 x i32> + %3 = sext <4 x i16> %2 to <4 x i32> + ret <4 x i32> %3 } define <2 x i64> @test_llvm_x86_sse41_pmovsxwq(<8 x i16>* %a) { -; CHECK-LABEL: test_llvm_x86_sse41_pmovsxwq -; SSE41: pmovsxwq (%rdi), %xmm0 -; AVX: vpmovsxwq (%rdi), %xmm0 +; SSE41-LABEL: test_llvm_x86_sse41_pmovsxwq: +; SSE41: ## BB#0: +; SSE41-NEXT: pmovsxwq (%rdi), %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: test_llvm_x86_sse41_pmovsxwq: +; AVX: ## BB#0: +; AVX-NEXT: vpmovsxwq (%rdi), %xmm0 +; AVX-NEXT: retq %1 = load <8 x i16>, <8 x i16>* %a, align 1 - %2 = call <2 x i64> @llvm.x86.sse41.pmovsxwq(<8 x i16> %1) - ret <2 x i64> %2 + %2 = shufflevector <8 x i16> %1, <8 x i16> undef, <2 x i32> + %3 = sext <2 x i16> %2 to <2 x i64> + ret <2 x i64> %3 } define <2 x i64> @test_llvm_x86_sse41_pmovsxdq(<4 x i32>* %a) { -; CHECK-LABEL: test_llvm_x86_sse41_pmovsxdq -; SSE41: pmovsxdq (%rdi), %xmm0 -; AVX: vpmovsxdq (%rdi), %xmm0 +; SSE41-LABEL: test_llvm_x86_sse41_pmovsxdq: +; SSE41: ## BB#0: +; SSE41-NEXT: pmovsxdq (%rdi), %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: test_llvm_x86_sse41_pmovsxdq: +; AVX: ## BB#0: +; AVX-NEXT: vpmovsxdq (%rdi), %xmm0 +; AVX-NEXT: retq %1 = load <4 x i32>, <4 x i32>* %a, align 1 - %2 = call <2 x i64> @llvm.x86.sse41.pmovsxdq(<4 x i32> %1) - ret <2 x i64> %2 + %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <2 x i32> + %3 = sext <2 x i32> %2 to <2 x i64> + ret <2 x i64> %3 } define <8 x i16> @test_llvm_x86_sse41_pmovzxbw(<16 x i8>* %a) { -; CHECK-LABEL: test_llvm_x86_sse41_pmovzxbw -; SSE41: pmovzxbw (%rdi), %xmm0 -; AVX: vpmovzxbw (%rdi), %xmm0 +; SSE41-LABEL: test_llvm_x86_sse41_pmovzxbw: +; SSE41: ## BB#0: +; SSE41-NEXT: pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero +; SSE41-NEXT: retq +; +; AVX-LABEL: test_llvm_x86_sse41_pmovzxbw: +; AVX: ## BB#0: +; AVX-NEXT: vpmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero +; AVX-NEXT: retq %1 = load <16 x i8>, <16 x i8>* %a, align 1 %2 = call <8 x i16> @llvm.x86.sse41.pmovzxbw(<16 x i8> %1) ret <8 x i16> %2 } define <4 x i32> @test_llvm_x86_sse41_pmovzxbd(<16 x i8>* %a) { -; CHECK-LABEL: test_llvm_x86_sse41_pmovzxbd -; SSE41: pmovzxbd (%rdi), %xmm0 -; AVX: vpmovzxbd (%rdi), %xmm0 +; SSE41-LABEL: test_llvm_x86_sse41_pmovzxbd: +; SSE41: ## BB#0: +; SSE41-NEXT: pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero +; SSE41-NEXT: retq +; +; AVX-LABEL: test_llvm_x86_sse41_pmovzxbd: +; AVX: ## BB#0: +; AVX-NEXT: vpmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero +; AVX-NEXT: retq %1 = load <16 x i8>, <16 x i8>* %a, align 1 %2 = call <4 x i32> @llvm.x86.sse41.pmovzxbd(<16 x i8> %1) ret <4 x i32> %2 } define <2 x i64> @test_llvm_x86_sse41_pmovzxbq(<16 x i8>* %a) { -; CHECK-LABEL: test_llvm_x86_sse41_pmovzxbq -; SSE41: pmovzxbq (%rdi), %xmm0 -; AVX: vpmovzxbq (%rdi), %xmm0 +; SSE41-LABEL: test_llvm_x86_sse41_pmovzxbq: +; SSE41: ## BB#0: +; SSE41-NEXT: pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero +; SSE41-NEXT: retq +; +; AVX-LABEL: test_llvm_x86_sse41_pmovzxbq: +; AVX: ## BB#0: +; AVX-NEXT: vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero +; AVX-NEXT: retq %1 = load <16 x i8>, <16 x i8>* %a, align 1 %2 = call <2 x i64> @llvm.x86.sse41.pmovzxbq(<16 x i8> %1) ret <2 x i64> %2 } define <4 x i32> @test_llvm_x86_sse41_pmovzxwd(<8 x i16>* %a) { -; CHECK-LABEL: test_llvm_x86_sse41_pmovzxwd -; SSE41: pmovzxwd (%rdi), %xmm0 -; AVX: vpmovzxwd (%rdi), %xmm0 +; SSE41-LABEL: test_llvm_x86_sse41_pmovzxwd: +; SSE41: ## BB#0: +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero +; SSE41-NEXT: retq +; +; AVX-LABEL: test_llvm_x86_sse41_pmovzxwd: +; AVX: ## BB#0: +; AVX-NEXT: vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero +; AVX-NEXT: retq %1 = load <8 x i16>, <8 x i16>* %a, align 1 %2 = call <4 x i32> @llvm.x86.sse41.pmovzxwd(<8 x i16> %1) ret <4 x i32> %2 } define <2 x i64> @test_llvm_x86_sse41_pmovzxwq(<8 x i16>* %a) { -; CHECK-LABEL: test_llvm_x86_sse41_pmovzxwq -; SSE41: pmovzxwq (%rdi), %xmm0 -; AVX: vpmovzxwq (%rdi), %xmm0 +; SSE41-LABEL: test_llvm_x86_sse41_pmovzxwq: +; SSE41: ## BB#0: +; SSE41-NEXT: pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero +; SSE41-NEXT: retq +; +; AVX-LABEL: test_llvm_x86_sse41_pmovzxwq: +; AVX: ## BB#0: +; AVX-NEXT: vpmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero +; AVX-NEXT: retq %1 = load <8 x i16>, <8 x i16>* %a, align 1 %2 = call <2 x i64> @llvm.x86.sse41.pmovzxwq(<8 x i16> %1) ret <2 x i64> %2 } define <2 x i64> @test_llvm_x86_sse41_pmovzxdq(<4 x i32>* %a) { -; CHECK-LABEL: test_llvm_x86_sse41_pmovzxdq -; SSE41: pmovzxdq (%rdi), %xmm0 -; AVX: vpmovzxdq (%rdi), %xmm0 +; SSE41-LABEL: test_llvm_x86_sse41_pmovzxdq: +; SSE41: ## BB#0: +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero +; SSE41-NEXT: retq +; +; AVX-LABEL: test_llvm_x86_sse41_pmovzxdq: +; AVX: ## BB#0: +; AVX-NEXT: vpmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero +; AVX-NEXT: retq %1 = load <4 x i32>, <4 x i32>* %a, align 1 %2 = call <2 x i64> @llvm.x86.sse41.pmovzxdq(<4 x i32> %1) ret <2 x i64> %2 @@ -115,9 +194,3 @@ declare <4 x i32> @llvm.x86.sse41.pmovzxwd(<8 x i16>) declare <2 x i64> @llvm.x86.sse41.pmovzxbq(<16 x i8>) declare <4 x i32> @llvm.x86.sse41.pmovzxbd(<16 x i8>) declare <8 x i16> @llvm.x86.sse41.pmovzxbw(<16 x i8>) -declare <2 x i64> @llvm.x86.sse41.pmovsxdq(<4 x i32>) -declare <2 x i64> @llvm.x86.sse41.pmovsxwq(<8 x i16>) -declare <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16>) -declare <2 x i64> @llvm.x86.sse41.pmovsxbq(<16 x i8>) -declare <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8>) -declare <8 x i16> @llvm.x86.sse41.pmovsxbw(<16 x i8>) diff --git a/test/CodeGen/X86/sse41.ll b/test/CodeGen/X86/sse41.ll index 8532c012aa9b..0a83a9753b81 100644 --- a/test/CodeGen/X86/sse41.ll +++ b/test/CodeGen/X86/sse41.ll @@ -31,49 +31,6 @@ define <16 x i8> @pinsrb_1(i8 %s, <16 x i8> %tmp) nounwind { ret <16 x i8> %tmp1 } -define <2 x i64> @pmovsxbd_1(i32* %p) nounwind { -; X32-LABEL: pmovsxbd_1: -; X32: ## BB#0: ## %entry -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: pmovsxbd (%eax), %xmm0 -; X32-NEXT: retl -; -; X64-LABEL: pmovsxbd_1: -; X64: ## BB#0: ## %entry -; X64-NEXT: pmovsxbd (%rdi), %xmm0 -; X64-NEXT: retq -entry: - %0 = load i32, i32* %p, align 4 - %1 = insertelement <4 x i32> undef, i32 %0, i32 0 - %2 = insertelement <4 x i32> %1, i32 0, i32 1 - %3 = insertelement <4 x i32> %2, i32 0, i32 2 - %4 = insertelement <4 x i32> %3, i32 0, i32 3 - %5 = bitcast <4 x i32> %4 to <16 x i8> - %6 = tail call <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8> %5) nounwind readnone - %7 = bitcast <4 x i32> %6 to <2 x i64> - ret <2 x i64> %7 -} - -define <2 x i64> @pmovsxwd_1(i64* %p) nounwind readonly { -; X32-LABEL: pmovsxwd_1: -; X32: ## BB#0: ## %entry -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: pmovsxwd (%eax), %xmm0 -; X32-NEXT: retl -; -; X64-LABEL: pmovsxwd_1: -; X64: ## BB#0: ## %entry -; X64-NEXT: pmovsxwd (%rdi), %xmm0 -; X64-NEXT: retq -entry: - %0 = load i64, i64* %p ; [#uses=1] - %tmp2 = insertelement <2 x i64> zeroinitializer, i64 %0, i32 0 ; <<2 x i64>> [#uses=1] - %1 = bitcast <2 x i64> %tmp2 to <8 x i16> ; <<8 x i16>> [#uses=1] - %2 = tail call <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16> %1) nounwind readnone ; <<4 x i32>> [#uses=1] - %3 = bitcast <4 x i32> %2 to <2 x i64> ; <<2 x i64>> [#uses=1] - ret <2 x i64> %3 -} - define <2 x i64> @pmovzxbq_1() nounwind { ; X32-LABEL: pmovzxbq_1: ; X32: ## BB#0: ## %entry @@ -94,8 +51,6 @@ entry: ret <2 x i64> %3 } -declare <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8>) nounwind readnone -declare <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16>) nounwind readnone declare <2 x i64> @llvm.x86.sse41.pmovzxbq(<16 x i8>) nounwind readnone define i32 @extractps_1(<4 x float> %v) nounwind { @@ -137,7 +92,7 @@ define float @ext_1(<4 x float> %v) nounwind { ; X32: ## BB#0: ; X32-NEXT: pushl %eax ; X32-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,1,2,3] -; X32-NEXT: addss LCPI7_0, %xmm0 +; X32-NEXT: addss LCPI5_0, %xmm0 ; X32-NEXT: movss %xmm0, (%esp) ; X32-NEXT: flds (%esp) ; X32-NEXT: popl %eax @@ -204,7 +159,7 @@ declare <4 x float> @llvm.x86.sse41.insertps(<4 x float>, <4 x float>, i32) noun define <4 x float> @blendps_not_insertps_1(<4 x float> %t1, float %t2) nounwind { ; X32-LABEL: blendps_not_insertps_1: ; X32: ## BB#0: -; X32-NEXT: movss {{.*#+}} xmm1 +; X32-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero ; X32-NEXT: blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] ; X32-NEXT: retl ; @@ -839,12 +794,12 @@ define <4 x float> @insertps_from_vector_load(<4 x float> %a, <4 x float>* nocap ; X32-LABEL: insertps_from_vector_load: ; X32: ## BB#0: ; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] +; X32-NEXT: insertps $48, (%{{...}}), {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] ; X32-NEXT: retl ; ; X64-LABEL: insertps_from_vector_load: ; X64: ## BB#0: -; X64-NEXT: insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] +; X64-NEXT: insertps $48, (%{{...}}), {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] ; X64-NEXT: retq %1 = load <4 x float>, <4 x float>* %pb, align 16 %2 = tail call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %a, <4 x float> %1, i32 48) @@ -857,12 +812,12 @@ define <4 x float> @insertps_from_vector_load_offset(<4 x float> %a, <4 x float> ; X32-LABEL: insertps_from_vector_load_offset: ; X32: ## BB#0: ; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: insertps {{.*#+}} xmm0 = xmm0[0,1],mem[1],xmm0[3] +; X32-NEXT: insertps $32, 4(%{{...}}), {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3] ; X32-NEXT: retl ; ; X64-LABEL: insertps_from_vector_load_offset: ; X64: ## BB#0: -; X64-NEXT: insertps {{.*#+}} xmm0 = xmm0[0,1],mem[1],xmm0[3] +; X64-NEXT: insertps $32, 4(%{{...}}), {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3] ; X64-NEXT: retq %1 = load <4 x float>, <4 x float>* %pb, align 16 %2 = tail call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %a, <4 x float> %1, i32 96) @@ -876,13 +831,13 @@ define <4 x float> @insertps_from_vector_load_offset_2(<4 x float> %a, <4 x floa ; X32-NEXT: movl {{[0-9]+}}(%esp), %eax ; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X32-NEXT: shll $4, %ecx -; X32-NEXT: insertps {{.*#+}} xmm0 = mem[3],xmm0[1,2,3] +; X32-NEXT: insertps $0, 12(%{{...}},%{{...}}), {{.*#+}} xmm0 = mem[0],xmm0[1,2,3] ; X32-NEXT: retl ; ; X64-LABEL: insertps_from_vector_load_offset_2: ; X64: ## BB#0: ; X64-NEXT: shlq $4, %rsi -; X64-NEXT: insertps {{.*#+}} xmm0 = mem[3],xmm0[1,2,3] +; X64-NEXT: insertps $0, 12(%{{...}},%{{...}}), {{.*#+}} xmm0 = mem[0],xmm0[1,2,3] ; X64-NEXT: retq %1 = getelementptr inbounds <4 x float>, <4 x float>* %pb, i64 %index %2 = load <4 x float>, <4 x float>* %1, align 16 @@ -1013,12 +968,12 @@ define <4 x float> @pr20087(<4 x float> %a, <4 x float> *%ptr) { ; X32-LABEL: pr20087: ; X32: ## BB#0: ; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: insertps {{.*#+}} xmm0 = xmm0[0],zero,xmm0[2],mem[2] +; X32-NEXT: insertps {{.*#+}} xmm0 = xmm0[0],zero,xmm0[2],mem[0] ; X32-NEXT: retl ; ; X64-LABEL: pr20087: ; X64: ## BB#0: -; X64-NEXT: insertps {{.*#+}} xmm0 = xmm0[0],zero,xmm0[2],mem[2] +; X64-NEXT: insertps {{.*#+}} xmm0 = xmm0[0],zero,xmm0[2],mem[0] ; X64-NEXT: retq %load = load <4 x float> , <4 x float> *%ptr %ret = shufflevector <4 x float> %load, <4 x float> %a, <4 x i32> diff --git a/test/CodeGen/X86/sse4a-intrinsics-fast-isel.ll b/test/CodeGen/X86/sse4a-intrinsics-fast-isel.ll new file mode 100644 index 000000000000..f93a16a5eb3d --- /dev/null +++ b/test/CodeGen/X86/sse4a-intrinsics-fast-isel.ll @@ -0,0 +1,98 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -fast-isel -mtriple=i386-unknown-unknown -mattr=+sse4a | FileCheck %s --check-prefix=ALL --check-prefix=X32 +; RUN: llc < %s -fast-isel -mtriple=x86_64-unknown-unknown -mattr=+sse4a | FileCheck %s --check-prefix=ALL --check-prefix=X64 + +; NOTE: This should use IR equivalent to what is generated by clang/test/CodeGen/sse4a-builtins.c + +define <2 x i64> @test_mm_extracti_si64(<2 x i64> %x) { +; X32-LABEL: test_mm_extracti_si64: +; X32: # BB#0: +; X32-NEXT: extrq $2, $3, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_extracti_si64: +; X64: # BB#0: +; X64-NEXT: extrq $2, $3, %xmm0 +; X64-NEXT: retq + %res = call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> %x, i8 3, i8 2) + ret <2 x i64> %res +} +declare <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64>, i8, i8) nounwind readnone + +define <2 x i64> @test_mm_extract_si64(<2 x i64> %x, <2 x i64> %y) { +; X32-LABEL: test_mm_extract_si64: +; X32: # BB#0: +; X32-NEXT: extrq %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_extract_si64: +; X64: # BB#0: +; X64-NEXT: extrq %xmm1, %xmm0 +; X64-NEXT: retq + %bc = bitcast <2 x i64> %y to <16 x i8> + %res = call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> %x, <16 x i8> %bc) + ret <2 x i64> %res +} +declare <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64>, <16 x i8>) nounwind readnone + +define <2 x i64> @test_mm_inserti_si64(<2 x i64> %x, <2 x i64> %y) { +; X32-LABEL: test_mm_inserti_si64: +; X32: # BB#0: +; X32-NEXT: insertq $6, $5, %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_inserti_si64: +; X64: # BB#0: +; X64-NEXT: insertq $6, $5, %xmm1, %xmm0 +; X64-NEXT: retq + %res = call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %x, <2 x i64> %y, i8 5, i8 6) + ret <2 x i64> %res +} +declare <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64>, <2 x i64>, i8, i8) nounwind readnone + +define <2 x i64> @test_mm_insert_si64(<2 x i64> %x, <2 x i64> %y) { +; X32-LABEL: test_mm_insert_si64: +; X32: # BB#0: +; X32-NEXT: insertq %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_insert_si64: +; X64: # BB#0: +; X64-NEXT: insertq %xmm1, %xmm0 +; X64-NEXT: retq + %res = call <2 x i64> @llvm.x86.sse4a.insertq(<2 x i64> %x, <2 x i64> %y) + ret <2 x i64> %res +} +declare <2 x i64> @llvm.x86.sse4a.insertq(<2 x i64>, <2 x i64>) nounwind readnone + +define void @test_stream_sd(i8* %p, <2 x double> %a) { +; X32-LABEL: test_stream_sd: +; X32: # BB#0: +; X32-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-NEXT: movntsd %xmm0, (%eax) +; X32-NEXT: retl +; +; X64-LABEL: test_stream_sd: +; X64: # BB#0: +; X64-NEXT: movntsd %xmm0, (%rdi) +; X64-NEXT: retq + call void @llvm.x86.sse4a.movnt.sd(i8* %p, <2 x double> %a) + ret void +} +declare void @llvm.x86.sse4a.movnt.sd(i8*, <2 x double>) nounwind readnone + +define void @test_mm_stream_ss(i8* %p, <4 x float> %a) { +; X32-LABEL: test_mm_stream_ss: +; X32: # BB#0: +; X32-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-NEXT: movntss %xmm0, (%eax) +; X32-NEXT: retl +; +; X64-LABEL: test_mm_stream_ss: +; X64: # BB#0: +; X64-NEXT: movntss %xmm0, (%rdi) +; X64-NEXT: retq + call void @llvm.x86.sse4a.movnt.ss(i8* %p, <4 x float> %a) + ret void +} +declare void @llvm.x86.sse4a.movnt.ss(i8*, <4 x float>) nounwind readnone diff --git a/test/CodeGen/X86/sse_partial_update.ll b/test/CodeGen/X86/sse_partial_update.ll index 377c3b7d6ead..8d61428420f6 100644 --- a/test/CodeGen/X86/sse_partial_update.ll +++ b/test/CodeGen/X86/sse_partial_update.ll @@ -90,3 +90,36 @@ entry: declare void @callee2(float, float) declare <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double>) nounwind readnone +define <2 x double> @load_fold_cvtss2sd_int(<4 x float> *%a) { +; CHECK-LABEL: load_fold_cvtss2sd_int: +; CHECK: movaps (%rdi), %xmm1 +; CHECK-NEXT: xorps %xmm0, %xmm0 +; CHECK-NEXT: cvtss2sd %xmm1, %xmm0 +; CHECK-NEXT: retq + %ld = load <4 x float>, <4 x float> *%a + %x = call <2 x double> @llvm.x86.sse2.cvtss2sd(<2 x double> , <4 x float> %ld) + ret <2 x double> %x +} + +define <2 x double> @load_fold_cvtss2sd_int_optsize(<4 x float> *%a) optsize { +; CHECK-LABEL: load_fold_cvtss2sd_int_optsize: +; CHECK: xorps %xmm0, %xmm0 +; CHECK-NEXT: cvtss2sd (%rdi), %xmm0 +; CHECK-NEXT: retq + %ld = load <4 x float>, <4 x float> *%a + %x = call <2 x double> @llvm.x86.sse2.cvtss2sd(<2 x double> , <4 x float> %ld) + ret <2 x double> %x +} + +define <2 x double> @load_fold_cvtss2sd_int_minsize(<4 x float> *%a) minsize { +; CHECK-LABEL: load_fold_cvtss2sd_int_minsize: +; CHECK: xorps %xmm0, %xmm0 +; CHECK-NEXT: cvtss2sd (%rdi), %xmm0 +; CHECK-NEXT: retq + %ld = load <4 x float>, <4 x float> *%a + %x = call <2 x double> @llvm.x86.sse2.cvtss2sd(<2 x double> , <4 x float> %ld) + ret <2 x double> %x +} + +declare <2 x double> @llvm.x86.sse2.cvtss2sd(<2 x double>, <4 x float>) nounwind readnone + diff --git a/test/CodeGen/X86/ssse3-intrinsics-fast-isel.ll b/test/CodeGen/X86/ssse3-intrinsics-fast-isel.ll new file mode 100644 index 000000000000..4f7ff20c6e0d --- /dev/null +++ b/test/CodeGen/X86/ssse3-intrinsics-fast-isel.ll @@ -0,0 +1,290 @@ +; RUN: llc < %s -fast-isel -mtriple=i386-unknown-unknown -mattr=ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=X32 +; RUN: llc < %s -fast-isel -mtriple=x86_64-unknown-unknown -mattr=ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=X64 + +; NOTE: This should use IR equivalent to what is generated by clang/test/CodeGen/ssse3-builtins.c + +define <2 x i64> @test_mm_abs_epi8(<2 x i64> %a0) { +; X32-LABEL: test_mm_abs_epi8: +; X32: # BB#0: +; X32-NEXT: pabsb %xmm0, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_abs_epi8: +; X64: # BB#0: +; X64-NEXT: pabsb %xmm0, %xmm0 +; X64-NEXT: retq + %arg = bitcast <2 x i64> %a0 to <16 x i8> + %call = call <16 x i8> @llvm.x86.ssse3.pabs.b.128(<16 x i8> %arg) + %res = bitcast <16 x i8> %call to <2 x i64> + ret <2 x i64> %res +} +declare <16 x i8> @llvm.x86.ssse3.pabs.b.128(<16 x i8>) nounwind readnone + +define <2 x i64> @test_mm_abs_epi16(<2 x i64> %a0) { +; X32-LABEL: test_mm_abs_epi16: +; X32: # BB#0: +; X32-NEXT: pabsw %xmm0, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_abs_epi16: +; X64: # BB#0: +; X64-NEXT: pabsw %xmm0, %xmm0 +; X64-NEXT: retq + %arg = bitcast <2 x i64> %a0 to <8 x i16> + %call = call <8 x i16> @llvm.x86.ssse3.pabs.w.128(<8 x i16> %arg) + %res = bitcast <8 x i16> %call to <2 x i64> + ret <2 x i64> %res +} +declare <8 x i16> @llvm.x86.ssse3.pabs.w.128(<8 x i16>) nounwind readnone + +define <2 x i64> @test_mm_abs_epi32(<2 x i64> %a0) { +; X32-LABEL: test_mm_abs_epi32: +; X32: # BB#0: +; X32-NEXT: pabsd %xmm0, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_abs_epi32: +; X64: # BB#0: +; X64-NEXT: pabsd %xmm0, %xmm0 +; X64-NEXT: retq + %arg = bitcast <2 x i64> %a0 to <4 x i32> + %call = call <4 x i32> @llvm.x86.ssse3.pabs.d.128(<4 x i32> %arg) + %res = bitcast <4 x i32> %call to <2 x i64> + ret <2 x i64> %res +} +declare <4 x i32> @llvm.x86.ssse3.pabs.d.128(<4 x i32>) nounwind readnone + +define <2 x i64> @test_mm_alignr_epi8(<2 x i64> %a0, <2 x i64> %a1) { +; X32-LABEL: test_mm_alignr_epi8: +; X32: # BB#0: +; X32-NEXT: palignr {{.*#}} xmm1 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1] +; X32-NEXT: movdqa %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_alignr_epi8: +; X64: # BB#0: +; X64-NEXT: palignr {{.*#}} xmm1 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1] +; X64-NEXT: movdqa %xmm1, %xmm0 +; X64-NEXT: retq + %arg0 = bitcast <2 x i64> %a0 to <16 x i8> + %arg1 = bitcast <2 x i64> %a1 to <16 x i8> + %shuf = shufflevector <16 x i8> %arg0, <16 x i8> %arg1, <16 x i32> + %res = bitcast <16 x i8> %shuf to <2 x i64> + ret <2 x i64> %res +} + +define <2 x i64> @test_mm_hadd_epi16(<2 x i64> %a0, <2 x i64> %a1) { +; X32-LABEL: test_mm_hadd_epi16: +; X32: # BB#0: +; X32-NEXT: phaddw %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_hadd_epi16: +; X64: # BB#0: +; X64-NEXT: phaddw %xmm1, %xmm0 +; X64-NEXT: retq + %arg0 = bitcast <2 x i64> %a0 to <8 x i16> + %arg1 = bitcast <2 x i64> %a1 to <8 x i16> + %call = call <8 x i16> @llvm.x86.ssse3.phadd.w.128(<8 x i16> %arg0, <8 x i16> %arg1) + %res = bitcast <8 x i16> %call to <2 x i64> + ret <2 x i64> %res +} +declare <8 x i16> @llvm.x86.ssse3.phadd.w.128(<8 x i16>, <8 x i16>) nounwind readnone + +define <2 x i64> @test_mm_hadd_epi32(<2 x i64> %a0, <2 x i64> %a1) { +; X32-LABEL: test_mm_hadd_epi32: +; X32: # BB#0: +; X32-NEXT: phaddd %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_hadd_epi32: +; X64: # BB#0: +; X64-NEXT: phaddd %xmm1, %xmm0 +; X64-NEXT: retq + %arg0 = bitcast <2 x i64> %a0 to <4 x i32> + %arg1 = bitcast <2 x i64> %a1 to <4 x i32> + %call = call <4 x i32> @llvm.x86.ssse3.phadd.d.128(<4 x i32> %arg0, <4 x i32> %arg1) + %res = bitcast <4 x i32> %call to <2 x i64> + ret <2 x i64> %res +} +declare <4 x i32> @llvm.x86.ssse3.phadd.d.128(<4 x i32>, <4 x i32>) nounwind readnone + +define <2 x i64> @test_mm_hadds_epi16(<2 x i64> %a0, <2 x i64> %a1) { +; X32-LABEL: test_mm_hadds_epi16: +; X32: # BB#0: +; X32-NEXT: phaddsw %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_hadds_epi16: +; X64: # BB#0: +; X64-NEXT: phaddsw %xmm1, %xmm0 +; X64-NEXT: retq + %arg0 = bitcast <2 x i64> %a0 to <8 x i16> + %arg1 = bitcast <2 x i64> %a1 to <8 x i16> + %call = call <8 x i16> @llvm.x86.ssse3.phadd.sw.128(<8 x i16> %arg0, <8 x i16> %arg1) + %res = bitcast <8 x i16> %call to <2 x i64> + ret <2 x i64> %res +} +declare <8 x i16> @llvm.x86.ssse3.phadd.sw.128(<8 x i16>, <8 x i16>) nounwind readnone + +define <2 x i64> @test_mm_hsub_epi16(<2 x i64> %a0, <2 x i64> %a1) { +; X32-LABEL: test_mm_hsub_epi16: +; X32: # BB#0: +; X32-NEXT: phsubw %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_hsub_epi16: +; X64: # BB#0: +; X64-NEXT: phsubw %xmm1, %xmm0 +; X64-NEXT: retq + %arg0 = bitcast <2 x i64> %a0 to <8 x i16> + %arg1 = bitcast <2 x i64> %a1 to <8 x i16> + %call = call <8 x i16> @llvm.x86.ssse3.phsub.w.128(<8 x i16> %arg0, <8 x i16> %arg1) + %res = bitcast <8 x i16> %call to <2 x i64> + ret <2 x i64> %res +} +declare <8 x i16> @llvm.x86.ssse3.phsub.w.128(<8 x i16>, <8 x i16>) nounwind readnone + +define <2 x i64> @test_mm_hsub_epi32(<2 x i64> %a0, <2 x i64> %a1) { +; X32-LABEL: test_mm_hsub_epi32: +; X32: # BB#0: +; X32-NEXT: phsubd %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_hsub_epi32: +; X64: # BB#0: +; X64-NEXT: phsubd %xmm1, %xmm0 +; X64-NEXT: retq + %arg0 = bitcast <2 x i64> %a0 to <4 x i32> + %arg1 = bitcast <2 x i64> %a1 to <4 x i32> + %call = call <4 x i32> @llvm.x86.ssse3.phsub.d.128(<4 x i32> %arg0, <4 x i32> %arg1) + %res = bitcast <4 x i32> %call to <2 x i64> + ret <2 x i64> %res +} +declare <4 x i32> @llvm.x86.ssse3.phsub.d.128(<4 x i32>, <4 x i32>) nounwind readnone + +define <2 x i64> @test_mm_hsubs_epi16(<2 x i64> %a0, <2 x i64> %a1) { +; X32-LABEL: test_mm_hsubs_epi16: +; X32: # BB#0: +; X32-NEXT: phsubsw %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_hsubs_epi16: +; X64: # BB#0: +; X64-NEXT: phsubsw %xmm1, %xmm0 +; X64-NEXT: retq + %arg0 = bitcast <2 x i64> %a0 to <8 x i16> + %arg1 = bitcast <2 x i64> %a1 to <8 x i16> + %call = call <8 x i16> @llvm.x86.ssse3.phsub.sw.128(<8 x i16> %arg0, <8 x i16> %arg1) + %res = bitcast <8 x i16> %call to <2 x i64> + ret <2 x i64> %res +} +declare <8 x i16> @llvm.x86.ssse3.phsub.sw.128(<8 x i16>, <8 x i16>) nounwind readnone + +define <2 x i64> @test_mm_maddubs_epi16(<2 x i64> %a0, <2 x i64> %a1) { +; X32-LABEL: test_mm_maddubs_epi16: +; X32: # BB#0: +; X32-NEXT: pmaddubsw %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_maddubs_epi16: +; X64: # BB#0: +; X64-NEXT: pmaddubsw %xmm1, %xmm0 +; X64-NEXT: retq + %arg0 = bitcast <2 x i64> %a0 to <16 x i8> + %arg1 = bitcast <2 x i64> %a1 to <16 x i8> + %call = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %arg0, <16 x i8> %arg1) + %res = bitcast <8 x i16> %call to <2 x i64> + ret <2 x i64> %res +} +declare <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8>, <16 x i8>) nounwind readnone + +define <2 x i64> @test_mm_mulhrs_epi16(<2 x i64> %a0, <2 x i64> %a1) { +; X32-LABEL: test_mm_mulhrs_epi16: +; X32: # BB#0: +; X32-NEXT: pmulhrsw %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_mulhrs_epi16: +; X64: # BB#0: +; X64-NEXT: pmulhrsw %xmm1, %xmm0 +; X64-NEXT: retq + %arg0 = bitcast <2 x i64> %a0 to <8 x i16> + %arg1 = bitcast <2 x i64> %a1 to <8 x i16> + %call = call <8 x i16> @llvm.x86.ssse3.pmul.hr.sw.128(<8 x i16> %arg0, <8 x i16> %arg1) + %res = bitcast <8 x i16> %call to <2 x i64> + ret <2 x i64> %res +} +declare <8 x i16> @llvm.x86.ssse3.pmul.hr.sw.128(<8 x i16>, <8 x i16>) nounwind readnone + +define <2 x i64> @test_mm_shuffle_epi8(<2 x i64> %a0, <2 x i64> %a1) { +; X32-LABEL: test_mm_shuffle_epi8: +; X32: # BB#0: +; X32-NEXT: pshufb %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_shuffle_epi8: +; X64: # BB#0: +; X64-NEXT: pshufb %xmm1, %xmm0 +; X64-NEXT: retq + %arg0 = bitcast <2 x i64> %a0 to <16 x i8> + %arg1 = bitcast <2 x i64> %a1 to <16 x i8> + %call = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %arg0, <16 x i8> %arg1) + %res = bitcast <16 x i8> %call to <2 x i64> + ret <2 x i64> %res +} +declare <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8>, <16 x i8>) nounwind readnone + +define <2 x i64> @test_mm_sign_epi8(<2 x i64> %a0, <2 x i64> %a1) { +; X32-LABEL: test_mm_sign_epi8: +; X32: # BB#0: +; X32-NEXT: psignb %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_sign_epi8: +; X64: # BB#0: +; X64-NEXT: psignb %xmm1, %xmm0 +; X64-NEXT: retq + %arg0 = bitcast <2 x i64> %a0 to <16 x i8> + %arg1 = bitcast <2 x i64> %a1 to <16 x i8> + %call = call <16 x i8> @llvm.x86.ssse3.psign.b.128(<16 x i8> %arg0, <16 x i8> %arg1) + %res = bitcast <16 x i8> %call to <2 x i64> + ret <2 x i64> %res +} +declare <16 x i8> @llvm.x86.ssse3.psign.b.128(<16 x i8>, <16 x i8>) nounwind readnone + +define <2 x i64> @test_mm_sign_epi16(<2 x i64> %a0, <2 x i64> %a1) { +; X32-LABEL: test_mm_sign_epi16: +; X32: # BB#0: +; X32-NEXT: psignw %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_sign_epi16: +; X64: # BB#0: +; X64-NEXT: psignw %xmm1, %xmm0 +; X64-NEXT: retq + %arg0 = bitcast <2 x i64> %a0 to <8 x i16> + %arg1 = bitcast <2 x i64> %a1 to <8 x i16> + %call = call <8 x i16> @llvm.x86.ssse3.psign.w.128(<8 x i16> %arg0, <8 x i16> %arg1) + %res = bitcast <8 x i16> %call to <2 x i64> + ret <2 x i64> %res +} +declare <8 x i16> @llvm.x86.ssse3.psign.w.128(<8 x i16>, <8 x i16>) nounwind readnone + +define <2 x i64> @test_mm_sign_epi32(<2 x i64> %a0, <2 x i64> %a1) { +; X32-LABEL: test_mm_sign_epi32: +; X32: # BB#0: +; X32-NEXT: psignd %xmm1, %xmm0 +; X32-NEXT: retl +; +; X64-LABEL: test_mm_sign_epi32: +; X64: # BB#0: +; X64-NEXT: psignd %xmm1, %xmm0 +; X64-NEXT: retq + %arg0 = bitcast <2 x i64> %a0 to <4 x i32> + %arg1 = bitcast <2 x i64> %a1 to <4 x i32> + %call = call <4 x i32> @llvm.x86.ssse3.psign.d.128(<4 x i32> %arg0, <4 x i32> %arg1) + %res = bitcast <4 x i32> %call to <2 x i64> + ret <2 x i64> %res +} +declare <4 x i32> @llvm.x86.ssse3.psign.d.128(<4 x i32>, <4 x i32>) nounwind readnone diff --git a/test/CodeGen/X86/stack-align-memcpy.ll b/test/CodeGen/X86/stack-align-memcpy.ll index 0cc3aa848891..129fb0c6b1f6 100644 --- a/test/CodeGen/X86/stack-align-memcpy.ll +++ b/test/CodeGen/X86/stack-align-memcpy.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -force-align-stack -mtriple i386-apple-darwin -mcpu=i486 | FileCheck %s +; RUN: llc < %s -stackrealign -mtriple i386-apple-darwin -mcpu=i486 | FileCheck %s %struct.foo = type { [88 x i8] } diff --git a/test/CodeGen/X86/stack-folding-adx-x86_64.ll b/test/CodeGen/X86/stack-folding-adx-x86_64.ll new file mode 100644 index 000000000000..5f109f09aa19 --- /dev/null +++ b/test/CodeGen/X86/stack-folding-adx-x86_64.ll @@ -0,0 +1,45 @@ +; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+adx < %s | FileCheck %s + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-unknown" + +; Stack reload folding tests. +; +; By including a nop call with sideeffects we can force a partial register spill of the +; relevant registers and check that the reload is correctly folded into the instruction. + +define i8 @stack_fold_addcarry_u32(i8 %a0, i32 %a1, i32 %a2, i8* %a3) { + ;CHECK-LABEL: stack_fold_addcarry_u32 + ;CHECK: adcxl {{-?[0-9]*}}(%rsp), %ecx {{.*#+}} 4-byte Folded Reload + %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"() + %2 = tail call i8 @llvm.x86.addcarry.u32(i8 %a0, i32 %a1, i32 %a2, i8* %a3) + ret i8 %2; +} +declare i8 @llvm.x86.addcarry.u32(i8, i32, i32, i8*) + +define i8 @stack_fold_addcarry_u64(i8 %a0, i64 %a1, i64 %a2, i8* %a3) { + ;CHECK-LABEL: stack_fold_addcarry_u64 + ;CHECK: adcxq {{-?[0-9]*}}(%rsp), %rcx {{.*#+}} 8-byte Folded Reload + %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"() + %2 = tail call i8 @llvm.x86.addcarry.u64(i8 %a0, i64 %a1, i64 %a2, i8* %a3) + ret i8 %2; +} +declare i8 @llvm.x86.addcarry.u64(i8, i64, i64, i8*) + +define i8 @stack_fold_addcarryx_u32(i8 %a0, i32 %a1, i32 %a2, i8* %a3) { + ;CHECK-LABEL: stack_fold_addcarryx_u32 + ;CHECK: adcxl {{-?[0-9]*}}(%rsp), %ecx {{.*#+}} 4-byte Folded Reload + %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"() + %2 = tail call i8 @llvm.x86.addcarryx.u32(i8 %a0, i32 %a1, i32 %a2, i8* %a3) + ret i8 %2; +} +declare i8 @llvm.x86.addcarryx.u32(i8, i32, i32, i8*) + +define i8 @stack_fold_addcarryx_u64(i8 %a0, i64 %a1, i64 %a2, i8* %a3) { + ;CHECK-LABEL: stack_fold_addcarryx_u64 + ;CHECK: adcxq {{-?[0-9]*}}(%rsp), %rcx {{.*#+}} 8-byte Folded Reload + %1 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"() + %2 = tail call i8 @llvm.x86.addcarryx.u64(i8 %a0, i64 %a1, i64 %a2, i8* %a3) + ret i8 %2; +} +declare i8 @llvm.x86.addcarryx.u64(i8, i64, i64, i8*) diff --git a/test/CodeGen/X86/stack-folding-fp-avx1.ll b/test/CodeGen/X86/stack-folding-fp-avx1.ll index 63aa742bdf01..b86ec0ea22ff 100644 --- a/test/CodeGen/X86/stack-folding-fp-avx1.ll +++ b/test/CodeGen/X86/stack-folding-fp-avx1.ll @@ -1,4 +1,4 @@ -; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx,+f16c < %s | FileCheck %s +; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx,+f16c < %s | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-unknown" @@ -946,7 +946,15 @@ define <8 x float> @stack_fold_insertf128(<4 x float> %a0, <4 x float> %a1) { ret <8 x float> %2 } -; TODO stack_fold_insertps +define <4 x float> @stack_fold_insertps(<4 x float> %a0, <4 x float> %a1) { + ;CHECK-LABEL: stack_fold_insertps + ;CHECK: vinsertps $17, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload + ;CHECK-NEXT: {{.*#+}} xmm0 = zero,mem[0],xmm0[2,3] + %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() + %2 = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %a0, <4 x float> %a1, i8 209) + ret <4 x float> %2 +} +declare <4 x float> @llvm.x86.sse41.insertps(<4 x float>, <4 x float>, i8) nounwind readnone define <2 x double> @stack_fold_maxpd(<2 x double> %a0, <2 x double> %a1) { ;CHECK-LABEL: stack_fold_maxpd @@ -1411,7 +1419,7 @@ declare <8 x float> @llvm.x86.avx.round.ps.256(<8 x float>, i32) nounwind readno define double @stack_fold_roundsd(double %a0) optsize { ;CHECK-LABEL: stack_fold_roundsd - ;CHECK: vroundsd $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload + ;CHECK: vroundsd $9, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() %2 = call double @llvm.floor.f64(double %a0) ret double %2 @@ -1423,7 +1431,7 @@ declare <2 x double> @llvm.x86.sse41.round.sd(<2 x double>, <2 x double>, i32) n define float @stack_fold_roundss(float %a0) optsize { ;CHECK-LABEL: stack_fold_roundss - ;CHECK: vroundss $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload + ;CHECK: vroundss $9, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() %2 = call float @llvm.floor.f32(float %a0) ret float %2 @@ -1494,7 +1502,7 @@ define <8 x float> @stack_fold_shufps_ymm(<8 x float> %a0, <8 x float> %a1) { ;CHECK-LABEL: stack_fold_shufps_ymm ;CHECK: vshufps $148, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = shufflevector <8 x float> %a0, <8 x float> %a1, <8 x i32> + %2 = shufflevector <8 x float> %a0, <8 x float> %a1, <8 x i32> ret <8 x float> %2 } diff --git a/test/CodeGen/X86/stack-folding-fp-sse42.ll b/test/CodeGen/X86/stack-folding-fp-sse42.ll index f9fcbaabdebb..105115bc7d25 100644 --- a/test/CodeGen/X86/stack-folding-fp-sse42.ll +++ b/test/CodeGen/X86/stack-folding-fp-sse42.ll @@ -1,4 +1,4 @@ -; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.2 < %s | FileCheck %s +; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 < %s | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-unknown" @@ -314,7 +314,7 @@ define i64 @stack_fold_cvtsd2si64_int(<2 x double> %a0) { } declare i64 @llvm.x86.sse2.cvtsd2si64(<2 x double>) nounwind readnone -define float @stack_fold_cvtsd2ss(double %a0) optsize { +define float @stack_fold_cvtsd2ss(double %a0) minsize { ;CHECK-LABEL: stack_fold_cvtsd2ss ;CHECK: cvtsd2ss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() @@ -331,7 +331,7 @@ define <4 x float> @stack_fold_cvtsd2ss_int(<2 x double> %a0) optsize { } declare <4 x float> @llvm.x86.sse2.cvtsd2ss(<4 x float>, <2 x double>) nounwind readnone -define double @stack_fold_cvtsi2sd(i32 %a0) optsize { +define double @stack_fold_cvtsi2sd(i32 %a0) minsize { ;CHECK-LABEL: stack_fold_cvtsi2sd ;CHECK: cvtsi2sdl {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"() @@ -365,7 +365,7 @@ define <2 x double> @stack_fold_cvtsi642sd_int(i64 %a0) { } declare <2 x double> @llvm.x86.sse2.cvtsi642sd(<2 x double>, i64) nounwind readnone -define float @stack_fold_cvtsi2ss(i32 %a0) optsize { +define float @stack_fold_cvtsi2ss(i32 %a0) minsize { ;CHECK-LABEL: stack_fold_cvtsi2ss ;CHECK: cvtsi2ssl {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"() @@ -399,7 +399,7 @@ define <4 x float> @stack_fold_cvtsi642ss_int(i64 %a0) { } declare <4 x float> @llvm.x86.sse.cvtsi642ss(<4 x float>, i64) nounwind readnone -define double @stack_fold_cvtss2sd(float %a0) optsize { +define double @stack_fold_cvtss2sd(float %a0) minsize { ;CHECK-LABEL: stack_fold_cvtss2sd ;CHECK: cvtss2sd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() @@ -637,7 +637,15 @@ define <4 x float> @stack_fold_hsubps(<4 x float> %a0, <4 x float> %a1) { } declare <4 x float> @llvm.x86.sse3.hsub.ps(<4 x float>, <4 x float>) nounwind readnone -; TODO stack_fold_insertps +define <4 x float> @stack_fold_insertps(<4 x float> %a0, <4 x float> %a1) { + ;CHECK-LABEL: stack_fold_insertps + ;CHECK: insertps $17, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload + ;CHECK-NEXT: {{.*#+}} xmm0 = zero,mem[0],xmm0[2,3] + %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() + %2 = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %a0, <4 x float> %a1, i8 209) + ret <4 x float> %2 +} +declare <4 x float> @llvm.x86.sse41.insertps(<4 x float>, <4 x float>, i8) nounwind readnone define <2 x double> @stack_fold_maxpd(<2 x double> %a0, <2 x double> %a1) { ;CHECK-LABEL: stack_fold_maxpd @@ -886,7 +894,7 @@ declare <4 x float> @llvm.x86.sse41.round.ps(<4 x float>, i32) nounwind readnone define double @stack_fold_roundsd(double %a0) optsize { ;CHECK-LABEL: stack_fold_roundsd - ;CHECK: roundsd $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload + ;CHECK: roundsd $9, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 8-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() %2 = call double @llvm.floor.f64(double %a0) ret double %2 @@ -896,9 +904,9 @@ declare double @llvm.floor.f64(double) nounwind readnone ; TODO stack_fold_roundsd_int declare <2 x double> @llvm.x86.sse41.round.sd(<2 x double>, <2 x double>, i32) nounwind readnone -define float @stack_fold_roundss(float %a0) optsize { +define float @stack_fold_roundss(float %a0) minsize { ;CHECK-LABEL: stack_fold_roundss - ;CHECK: roundss $1, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload + ;CHECK: roundss $9, {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() %2 = call float @llvm.floor.f32(float %a0) ret float %2 @@ -968,7 +976,7 @@ declare double @llvm.sqrt.f64(double) nounwind readnone ; TODO stack_fold_sqrtsd_int declare <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double>) nounwind readnone -define float @stack_fold_sqrtss(float %a0) optsize { +define float @stack_fold_sqrtss(float %a0) minsize { ;CHECK-LABEL: stack_fold_sqrtss ;CHECK: sqrtss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 4-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() diff --git a/test/CodeGen/X86/stack-folding-int-avx1.ll b/test/CodeGen/X86/stack-folding-int-avx1.ll index fec297d5e9d4..15ffb1d2dcc5 100644 --- a/test/CodeGen/X86/stack-folding-int-avx1.ll +++ b/test/CodeGen/X86/stack-folding-int-avx1.ll @@ -1,4 +1,4 @@ -; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx,+aes,+pclmul < %s | FileCheck %s +; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx,+aes,+pclmul < %s | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-unknown" @@ -671,55 +671,55 @@ define <4 x i32> @stack_fold_pmovsxbd(<16 x i8> %a0) { ;CHECK-LABEL: stack_fold_pmovsxbd ;CHECK: vpmovsxbd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8> %a0) - ret <4 x i32> %2 + %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <4 x i32> + %3 = sext <4 x i8> %2 to <4 x i32> + ret <4 x i32> %3 } -declare <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8>) nounwind readnone define <2 x i64> @stack_fold_pmovsxbq(<16 x i8> %a0) { ;CHECK-LABEL: stack_fold_pmovsxbq - ;CHECK: pmovsxbq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload + ;CHECK: vpmovsxbq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <2 x i64> @llvm.x86.sse41.pmovsxbq(<16 x i8> %a0) - ret <2 x i64> %2 + %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <2 x i32> + %3 = sext <2 x i8> %2 to <2 x i64> + ret <2 x i64> %3 } -declare <2 x i64> @llvm.x86.sse41.pmovsxbq(<16 x i8>) nounwind readnone define <8 x i16> @stack_fold_pmovsxbw(<16 x i8> %a0) { ;CHECK-LABEL: stack_fold_pmovsxbw ;CHECK: vpmovsxbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <8 x i16> @llvm.x86.sse41.pmovsxbw(<16 x i8> %a0) - ret <8 x i16> %2 + %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <8 x i32> + %3 = sext <8 x i8> %2 to <8 x i16> + ret <8 x i16> %3 } -declare <8 x i16> @llvm.x86.sse41.pmovsxbw(<16 x i8>) nounwind readnone define <2 x i64> @stack_fold_pmovsxdq(<4 x i32> %a0) { ;CHECK-LABEL: stack_fold_pmovsxdq ;CHECK: vpmovsxdq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <2 x i64> @llvm.x86.sse41.pmovsxdq(<4 x i32> %a0) - ret <2 x i64> %2 + %2 = shufflevector <4 x i32> %a0, <4 x i32> undef, <2 x i32> + %3 = sext <2 x i32> %2 to <2 x i64> + ret <2 x i64> %3 } -declare <2 x i64> @llvm.x86.sse41.pmovsxdq(<4 x i32>) nounwind readnone define <4 x i32> @stack_fold_pmovsxwd(<8 x i16> %a0) { ;CHECK-LABEL: stack_fold_pmovsxwd ;CHECK: vpmovsxwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16> %a0) - ret <4 x i32> %2 + %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> + %3 = sext <4 x i16> %2 to <4 x i32> + ret <4 x i32> %3 } -declare <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16>) nounwind readnone define <2 x i64> @stack_fold_pmovsxwq(<8 x i16> %a0) { ;CHECK-LABEL: stack_fold_pmovsxwq ;CHECK: vpmovsxwq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <2 x i64> @llvm.x86.sse41.pmovsxwq(<8 x i16> %a0) - ret <2 x i64> %2 + %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <2 x i32> + %3 = sext <2 x i16> %2 to <2 x i64> + ret <2 x i64> %3 } -declare <2 x i64> @llvm.x86.sse41.pmovsxwq(<8 x i16>) nounwind readnone define <4 x i32> @stack_fold_pmovzxbd(<16 x i8> %a0) { ;CHECK-LABEL: stack_fold_pmovzxbd diff --git a/test/CodeGen/X86/stack-folding-int-avx2.ll b/test/CodeGen/X86/stack-folding-int-avx2.ll index a164fbbc7a6a..235a10ed4678 100644 --- a/test/CodeGen/X86/stack-folding-int-avx2.ll +++ b/test/CodeGen/X86/stack-folding-int-avx2.ll @@ -1,4 +1,4 @@ -; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 < %s | FileCheck %s +; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx2 < %s | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-unknown" @@ -12,7 +12,7 @@ define <4 x double> @stack_fold_broadcastsd_ymm(<2 x double> %a0) { ;CHECK-LABEL: stack_fold_broadcastsd_ymm ;CHECK: vbroadcastsd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <4 x double> @llvm.x86.avx2.vbroadcast.sd.pd.256(<2 x double> %a0) + %2 = shufflevector <2 x double> %a0, <2 x double> undef, <4 x i32> zeroinitializer ; fadd forces execution domain %3 = fadd <4 x double> %2, ret <4 x double> %3 @@ -23,7 +23,7 @@ define <4 x float> @stack_fold_broadcastss(<4 x float> %a0) { ;CHECK-LABEL: stack_fold_broadcastss ;CHECK: vbroadcastss {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <4 x float> @llvm.x86.avx2.vbroadcast.ss.ps(<4 x float> %a0) + %2 = shufflevector <4 x float> %a0, <4 x float> undef, <4 x i32> zeroinitializer ; fadd forces execution domain %3 = fadd <4 x float> %2, ret <4 x float> %3 @@ -34,7 +34,7 @@ define <8 x float> @stack_fold_broadcastss_ymm(<4 x float> %a0) { ;CHECK-LABEL: stack_fold_broadcastss_ymm ;CHECK: vbroadcastss {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <8 x float> @llvm.x86.avx2.vbroadcast.ss.ps.256(<4 x float> %a0) + %2 = shufflevector <4 x float> %a0, <4 x float> undef, <8 x i32> zeroinitializer ; fadd forces execution domain %3 = fadd <8 x float> %2, ret <8 x float> %3 @@ -286,81 +286,73 @@ define <16 x i8> @stack_fold_pbroadcastb(<16 x i8> %a0) { ;CHECK-LABEL: stack_fold_pbroadcastb ;CHECK: vpbroadcastb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <16 x i8> @llvm.x86.avx2.pbroadcastb.128(<16 x i8> %a0) + %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <16 x i32> zeroinitializer ret <16 x i8> %2 } -declare <16 x i8> @llvm.x86.avx2.pbroadcastb.128(<16 x i8>) nounwind readonly define <32 x i8> @stack_fold_pbroadcastb_ymm(<16 x i8> %a0) { ;CHECK-LABEL: stack_fold_pbroadcastb_ymm ;CHECK: vpbroadcastb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <32 x i8> @llvm.x86.avx2.pbroadcastb.256(<16 x i8> %a0) + %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <32 x i32> zeroinitializer ret <32 x i8> %2 } -declare <32 x i8> @llvm.x86.avx2.pbroadcastb.256(<16 x i8>) nounwind readonly define <4 x i32> @stack_fold_pbroadcastd(<4 x i32> %a0) { ;CHECK-LABEL: stack_fold_pbroadcastd ;CHECK: vpbroadcastd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <4 x i32> @llvm.x86.avx2.pbroadcastd.128(<4 x i32> %a0) + %2 = shufflevector <4 x i32> %a0, <4 x i32> undef, <4 x i32> zeroinitializer ; add forces execution domain %3 = add <4 x i32> %2, ret <4 x i32> %3 } -declare <4 x i32> @llvm.x86.avx2.pbroadcastd.128(<4 x i32>) nounwind readonly define <8 x i32> @stack_fold_pbroadcastd_ymm(<4 x i32> %a0) { ;CHECK-LABEL: stack_fold_pbroadcastd_ymm ;CHECK: vpbroadcastd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <8 x i32> @llvm.x86.avx2.pbroadcastd.256(<4 x i32> %a0) + %2 = shufflevector <4 x i32> %a0, <4 x i32> undef, <8 x i32> zeroinitializer ; add forces execution domain %3 = add <8 x i32> %2, ret <8 x i32> %3 } -declare <8 x i32> @llvm.x86.avx2.pbroadcastd.256(<4 x i32>) nounwind readonly define <2 x i64> @stack_fold_pbroadcastq(<2 x i64> %a0) { ;CHECK-LABEL: stack_fold_pbroadcastq ;CHECK: vpbroadcastq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <2 x i64> @llvm.x86.avx2.pbroadcastq.128(<2 x i64> %a0) + %2 = shufflevector <2 x i64> %a0, <2 x i64> undef, <2 x i32> zeroinitializer ; add forces execution domain %3 = add <2 x i64> %2, ret <2 x i64> %3 } -declare <2 x i64> @llvm.x86.avx2.pbroadcastq.128(<2 x i64>) nounwind readonly define <4 x i64> @stack_fold_pbroadcastq_ymm(<2 x i64> %a0) { ;CHECK-LABEL: stack_fold_pbroadcastq_ymm ;CHECK: vpbroadcastq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <4 x i64> @llvm.x86.avx2.pbroadcastq.256(<2 x i64> %a0) + %2 = shufflevector <2 x i64> %a0, <2 x i64> undef, <4 x i32> zeroinitializer ; add forces execution domain %3 = add <4 x i64> %2, ret <4 x i64> %3 } -declare <4 x i64> @llvm.x86.avx2.pbroadcastq.256(<2 x i64>) nounwind readonly define <8 x i16> @stack_fold_pbroadcastw(<8 x i16> %a0) { ;CHECK-LABEL: stack_fold_pbroadcastw ;CHECK: vpbroadcastw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <8 x i16> @llvm.x86.avx2.pbroadcastw.128(<8 x i16> %a0) + %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> zeroinitializer ret <8 x i16> %2 } -declare <8 x i16> @llvm.x86.avx2.pbroadcastw.128(<8 x i16>) nounwind readonly define <16 x i16> @stack_fold_pbroadcastw_ymm(<8 x i16> %a0) { ;CHECK-LABEL: stack_fold_pbroadcastw_ymm ;CHECK: vpbroadcastw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <16 x i16> @llvm.x86.avx2.pbroadcastw.256(<8 x i16> %a0) + %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <16 x i32> zeroinitializer ret <16 x i16> %2 } -declare <16 x i16> @llvm.x86.avx2.pbroadcastw.256(<8 x i16>) nounwind readonly define <32 x i8> @stack_fold_pcmpeqb(<32 x i8> %a0, <32 x i8> %a1) { ;CHECK-LABEL: stack_fold_pcmpeqb @@ -455,28 +447,28 @@ declare <8 x i32> @llvm.x86.avx2.permd(<8 x i32>, <8 x i32>) nounwind readonly define <4 x double> @stack_fold_permpd(<4 x double> %a0) { ;CHECK-LABEL: stack_fold_permpd - ;CHECK: vpermpd $255, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload + ;CHECK: vpermpd $235, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = shufflevector <4 x double> %a0, <4 x double> undef, <4 x i32> + %2 = shufflevector <4 x double> %a0, <4 x double> undef, <4 x i32> ; fadd forces execution domain %3 = fadd <4 x double> %2, ret <4 x double> %3 } -define <8 x float> @stack_fold_permps(<8 x float> %a0, <8 x float> %a1) { +define <8 x float> @stack_fold_permps(<8 x i32> %a0, <8 x float> %a1) { ;CHECK-LABEL: stack_fold_permps ;CHECK: vpermps {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a1, <8 x float> %a0) + %2 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a1, <8 x i32> %a0) ret <8 x float> %2 } -declare <8 x float> @llvm.x86.avx2.permps(<8 x float>, <8 x float>) nounwind readonly +declare <8 x float> @llvm.x86.avx2.permps(<8 x float>, <8 x i32>) nounwind readonly define <4 x i64> @stack_fold_permq(<4 x i64> %a0) { ;CHECK-LABEL: stack_fold_permq - ;CHECK: vpermq $255, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload + ;CHECK: vpermq $235, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = shufflevector <4 x i64> %a0, <4 x i64> undef, <4 x i32> + %2 = shufflevector <4 x i64> %a0, <4 x i64> undef, <4 x i32> ; add forces execution domain %3 = add <4 x i64> %2, ret <4 x i64> %3 @@ -684,28 +676,25 @@ define <16 x i16> @stack_fold_pmovsxbw(<16 x i8> %a0) { ;CHECK-LABEL: stack_fold_pmovsxbw ;CHECK: vpmovsxbw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <16 x i16> @llvm.x86.avx2.pmovsxbw(<16 x i8> %a0) + %2 = sext <16 x i8> %a0 to <16 x i16> ret <16 x i16> %2 } -declare <16 x i16> @llvm.x86.avx2.pmovsxbw(<16 x i8>) nounwind readnone define <4 x i64> @stack_fold_pmovsxdq(<4 x i32> %a0) { ;CHECK-LABEL: stack_fold_pmovsxdq ;CHECK: vpmovsxdq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <4 x i64> @llvm.x86.avx2.pmovsxdq(<4 x i32> %a0) + %2 = sext <4 x i32> %a0 to <4 x i64> ret <4 x i64> %2 } -declare <4 x i64> @llvm.x86.avx2.pmovsxdq(<4 x i32>) nounwind readnone define <8 x i32> @stack_fold_pmovsxwd(<8 x i16> %a0) { ;CHECK-LABEL: stack_fold_pmovsxwd ;CHECK: vpmovsxwd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <8 x i32> @llvm.x86.avx2.pmovsxwd(<8 x i16> %a0) + %2 = sext <8 x i16> %a0 to <8 x i32> ret <8 x i32> %2 } -declare <8 x i32> @llvm.x86.avx2.pmovsxwd(<8 x i16>) nounwind readnone define <4 x i64> @stack_fold_pmovsxwq(<8 x i16> %a0) { ;CHECK-LABEL: stack_fold_pmovsxwq diff --git a/test/CodeGen/X86/stack-folding-int-sse42.ll b/test/CodeGen/X86/stack-folding-int-sse42.ll index e814ae6df501..f732607851fc 100644 --- a/test/CodeGen/X86/stack-folding-int-sse42.ll +++ b/test/CodeGen/X86/stack-folding-int-sse42.ll @@ -1,4 +1,4 @@ -; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.2,+aes,+pclmul < %s | FileCheck %s +; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+sse4.2,+aes,+pclmul < %s | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-unknown" @@ -698,55 +698,55 @@ define <4 x i32> @stack_fold_pmovsxbd(<16 x i8> %a0) { ;CHECK-LABEL: stack_fold_pmovsxbd ;CHECK: pmovsxbd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8> %a0) - ret <4 x i32> %2 + %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <4 x i32> + %3 = sext <4 x i8> %2 to <4 x i32> + ret <4 x i32> %3 } -declare <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8>) nounwind readnone define <2 x i64> @stack_fold_pmovsxbq(<16 x i8> %a0) { ;CHECK-LABEL: stack_fold_pmovsxbq ;CHECK: pmovsxbq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <2 x i64> @llvm.x86.sse41.pmovsxbq(<16 x i8> %a0) - ret <2 x i64> %2 + %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <2 x i32> + %3 = sext <2 x i8> %2 to <2 x i64> + ret <2 x i64> %3 } -declare <2 x i64> @llvm.x86.sse41.pmovsxbq(<16 x i8>) nounwind readnone define <8 x i16> @stack_fold_pmovsxbw(<16 x i8> %a0) { ;CHECK-LABEL: stack_fold_pmovsxbw ;CHECK: pmovsxbw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <8 x i16> @llvm.x86.sse41.pmovsxbw(<16 x i8> %a0) - ret <8 x i16> %2 + %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <8 x i32> + %3 = sext <8 x i8> %2 to <8 x i16> + ret <8 x i16> %3 } -declare <8 x i16> @llvm.x86.sse41.pmovsxbw(<16 x i8>) nounwind readnone define <2 x i64> @stack_fold_pmovsxdq(<4 x i32> %a0) { ;CHECK-LABEL: stack_fold_pmovsxdq ;CHECK: pmovsxdq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <2 x i64> @llvm.x86.sse41.pmovsxdq(<4 x i32> %a0) - ret <2 x i64> %2 + %2 = shufflevector <4 x i32> %a0, <4 x i32> undef, <2 x i32> + %3 = sext <2 x i32> %2 to <2 x i64> + ret <2 x i64> %3 } -declare <2 x i64> @llvm.x86.sse41.pmovsxdq(<4 x i32>) nounwind readnone define <4 x i32> @stack_fold_pmovsxwd(<8 x i16> %a0) { ;CHECK-LABEL: stack_fold_pmovsxwd ;CHECK: pmovsxwd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16> %a0) - ret <4 x i32> %2 + %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> + %3 = sext <4 x i16> %2 to <4 x i32> + ret <4 x i32> %3 } -declare <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16>) nounwind readnone define <2 x i64> @stack_fold_pmovsxwq(<8 x i16> %a0) { ;CHECK-LABEL: stack_fold_pmovsxwq ;CHECK: pmovsxwq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() - %2 = call <2 x i64> @llvm.x86.sse41.pmovsxwq(<8 x i16> %a0) - ret <2 x i64> %2 + %2 = shufflevector <8 x i16> %a0, <8 x i16> undef, <2 x i32> + %3 = sext <2 x i16> %2 to <2 x i64> + ret <2 x i64> %3 } -declare <2 x i64> @llvm.x86.sse41.pmovsxwq(<8 x i16>) nounwind readnone define <4 x i32> @stack_fold_pmovzxbd(<16 x i8> %a0) { ;CHECK-LABEL: stack_fold_pmovzxbd diff --git a/test/CodeGen/X86/stack-folding-mmx.ll b/test/CodeGen/X86/stack-folding-mmx.ll index 8a5d4e2770dc..3b1a4956726f 100644 --- a/test/CodeGen/X86/stack-folding-mmx.ll +++ b/test/CodeGen/X86/stack-folding-mmx.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+mmx,+sse2 | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+mmx,+ssse3 | FileCheck %s define x86_mmx @stack_fold_cvtpd2pi(<2 x double> %a0) { ;CHECK-LABEL: stack_fold_cvtpd2pi @@ -59,6 +59,33 @@ declare x86_mmx @llvm.x86.sse.cvttps2pi(<4 x float>) nounwind readnone ; TODO stack_fold_movq_load ; TODO stack_fold_movq_store +define x86_mmx @stack_fold_pabsb(x86_mmx %a0) { + ;CHECK-LABEL: stack_fold_pabsb + ;CHECK: pabsb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload + %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"() + %2 = call x86_mmx @llvm.x86.ssse3.pabs.b(x86_mmx %a0) nounwind readnone + ret x86_mmx %2 +} +declare x86_mmx @llvm.x86.ssse3.pabs.b(x86_mmx) nounwind readnone + +define x86_mmx @stack_fold_pabsd(x86_mmx %a0) { + ;CHECK-LABEL: stack_fold_pabsd + ;CHECK: pabsd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload + %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"() + %2 = call x86_mmx @llvm.x86.ssse3.pabs.d(x86_mmx %a0) nounwind readnone + ret x86_mmx %2 +} +declare x86_mmx @llvm.x86.ssse3.pabs.d(x86_mmx) nounwind readnone + +define x86_mmx @stack_fold_pabsw(x86_mmx %a0) { + ;CHECK-LABEL: stack_fold_pabsw + ;CHECK: pabsw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload + %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"() + %2 = call x86_mmx @llvm.x86.ssse3.pabs.w(x86_mmx %a0) nounwind readnone + ret x86_mmx %2 +} +declare x86_mmx @llvm.x86.ssse3.pabs.w(x86_mmx) nounwind readnone + define x86_mmx @stack_fold_packssdw(x86_mmx %a, x86_mmx %b) { ;CHECK-LABEL: stack_fold_packssdw ;CHECK: packssdw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload @@ -158,6 +185,15 @@ define x86_mmx @stack_fold_paddw(x86_mmx %a, x86_mmx %b) { } declare x86_mmx @llvm.x86.mmx.padd.w(x86_mmx, x86_mmx) nounwind readnone +define x86_mmx @stack_fold_palignr(x86_mmx %a, x86_mmx %b) { + ;CHECK-LABEL: stack_fold_palignr + ;CHECK: palignr $1, {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload + %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"() + %2 = call x86_mmx @llvm.x86.mmx.palignr.b(x86_mmx %a, x86_mmx %b, i8 1) nounwind readnone + ret x86_mmx %2 +} +declare x86_mmx @llvm.x86.mmx.palignr.b(x86_mmx, x86_mmx, i8) nounwind readnone + define x86_mmx @stack_fold_pand(x86_mmx %a, x86_mmx %b) { ;CHECK-LABEL: stack_fold_pand ;CHECK: pand {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload @@ -248,8 +284,71 @@ define x86_mmx @stack_fold_pcmpgtw(x86_mmx %a, x86_mmx %b) { } declare x86_mmx @llvm.x86.mmx.pcmpgt.w(x86_mmx, x86_mmx) nounwind readnone +define x86_mmx @stack_fold_phaddd(x86_mmx %a, x86_mmx %b) { + ;CHECK-LABEL: stack_fold_phaddd + ;CHECK: phaddd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload + %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"() + %2 = call x86_mmx @llvm.x86.ssse3.phadd.d(x86_mmx %a, x86_mmx %b) nounwind readnone + ret x86_mmx %2 +} +declare x86_mmx @llvm.x86.ssse3.phadd.d(x86_mmx, x86_mmx) nounwind readnone + +define x86_mmx @stack_fold_phaddsw(x86_mmx %a, x86_mmx %b) { + ;CHECK-LABEL: stack_fold_phaddsw + ;CHECK: phaddsw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload + %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"() + %2 = call x86_mmx @llvm.x86.ssse3.phadd.sw(x86_mmx %a, x86_mmx %b) nounwind readnone + ret x86_mmx %2 +} +declare x86_mmx @llvm.x86.ssse3.phadd.sw(x86_mmx, x86_mmx) nounwind readnone + +define x86_mmx @stack_fold_phaddw(x86_mmx %a, x86_mmx %b) { + ;CHECK-LABEL: stack_fold_phaddw + ;CHECK: phaddw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload + %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"() + %2 = call x86_mmx @llvm.x86.ssse3.phadd.w(x86_mmx %a, x86_mmx %b) nounwind readnone + ret x86_mmx %2 +} +declare x86_mmx @llvm.x86.ssse3.phadd.w(x86_mmx, x86_mmx) nounwind readnone + +define x86_mmx @stack_fold_phsubd(x86_mmx %a, x86_mmx %b) { + ;CHECK-LABEL: stack_fold_phsubd + ;CHECK: phsubd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload + %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"() + %2 = call x86_mmx @llvm.x86.ssse3.phsub.d(x86_mmx %a, x86_mmx %b) nounwind readnone + ret x86_mmx %2 +} +declare x86_mmx @llvm.x86.ssse3.phsub.d(x86_mmx, x86_mmx) nounwind readnone + +define x86_mmx @stack_fold_phsubsw(x86_mmx %a, x86_mmx %b) { + ;CHECK-LABEL: stack_fold_phsubsw + ;CHECK: phsubsw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload + %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"() + %2 = call x86_mmx @llvm.x86.ssse3.phsub.sw(x86_mmx %a, x86_mmx %b) nounwind readnone + ret x86_mmx %2 +} +declare x86_mmx @llvm.x86.ssse3.phsub.sw(x86_mmx, x86_mmx) nounwind readnone + +define x86_mmx @stack_fold_phsubw(x86_mmx %a, x86_mmx %b) { + ;CHECK-LABEL: stack_fold_phsubw + ;CHECK: phsubw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload + %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"() + %2 = call x86_mmx @llvm.x86.ssse3.phsub.w(x86_mmx %a, x86_mmx %b) nounwind readnone + ret x86_mmx %2 +} +declare x86_mmx @llvm.x86.ssse3.phsub.w(x86_mmx, x86_mmx) nounwind readnone + ; TODO stack_fold_pinsrw +define x86_mmx @stack_fold_pmaddubsw(x86_mmx %a, x86_mmx %b) { + ;CHECK-LABEL: stack_fold_pmaddubsw + ;CHECK: pmaddubsw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload + %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"() + %2 = call x86_mmx @llvm.x86.ssse3.pmadd.ub.sw(x86_mmx %a, x86_mmx %b) nounwind readnone + ret x86_mmx %2 +} +declare x86_mmx @llvm.x86.ssse3.pmadd.ub.sw(x86_mmx, x86_mmx) nounwind readnone + define x86_mmx @stack_fold_pmaddwd(x86_mmx %a, x86_mmx %b) { ;CHECK-LABEL: stack_fold_pmaddwd ;CHECK: pmaddwd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload @@ -295,6 +394,15 @@ define x86_mmx @stack_fold_pminub(x86_mmx %a, x86_mmx %b) { } declare x86_mmx @llvm.x86.mmx.pminu.b(x86_mmx, x86_mmx) nounwind readnone +define x86_mmx @stack_fold_pmulhrsw(x86_mmx %a, x86_mmx %b) { + ;CHECK-LABEL: stack_fold_pmulhrsw + ;CHECK: pmulhrsw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload + %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"() + %2 = call x86_mmx @llvm.x86.ssse3.pmul.hr.sw(x86_mmx %a, x86_mmx %b) nounwind readnone + ret x86_mmx %2 +} +declare x86_mmx @llvm.x86.ssse3.pmul.hr.sw(x86_mmx, x86_mmx) nounwind readnone + define x86_mmx @stack_fold_pmulhuw(x86_mmx %a, x86_mmx %b) { ;CHECK-LABEL: stack_fold_pmulhuw ;CHECK: pmulhuw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload @@ -349,7 +457,16 @@ define x86_mmx @stack_fold_psadbw(x86_mmx %a, x86_mmx %b) { } declare x86_mmx @llvm.x86.mmx.psad.bw(x86_mmx, x86_mmx) nounwind readnone -define x86_mmx @stack_fold_pshufw(x86_mmx %a, x86_mmx %b) { +define x86_mmx @stack_fold_pshufb(x86_mmx %a, x86_mmx %b) { + ;CHECK-LABEL: stack_fold_pshufb + ;CHECK: pshufb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload + %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"() + %2 = call x86_mmx @llvm.x86.ssse3.pshuf.b(x86_mmx %a, x86_mmx %b) nounwind readnone + ret x86_mmx %2 +} +declare x86_mmx @llvm.x86.ssse3.pshuf.b(x86_mmx, x86_mmx) nounwind readnone + +define x86_mmx @stack_fold_pshufw(x86_mmx %a) { ;CHECK-LABEL: stack_fold_pshufw ;CHECK: pshufw $1, {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm1},~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"() @@ -358,6 +475,33 @@ define x86_mmx @stack_fold_pshufw(x86_mmx %a, x86_mmx %b) { } declare x86_mmx @llvm.x86.sse.pshuf.w(x86_mmx, i8) nounwind readnone +define x86_mmx @stack_fold_psignb(x86_mmx %a0, x86_mmx %a1) { + ;CHECK-LABEL: stack_fold_psignb + ;CHECK: psignb {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload + %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"() + %2 = call x86_mmx @llvm.x86.ssse3.psign.b(x86_mmx %a0, x86_mmx %a1) nounwind readnone + ret x86_mmx %2 +} +declare x86_mmx @llvm.x86.ssse3.psign.b(x86_mmx, x86_mmx) nounwind readnone + +define x86_mmx @stack_fold_psignd(x86_mmx %a0, x86_mmx %a1) { + ;CHECK-LABEL: stack_fold_psignd + ;CHECK: psignd {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload + %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"() + %2 = call x86_mmx @llvm.x86.ssse3.psign.d(x86_mmx %a0, x86_mmx %a1) nounwind readnone + ret x86_mmx %2 +} +declare x86_mmx @llvm.x86.ssse3.psign.d(x86_mmx, x86_mmx) nounwind readnone + +define x86_mmx @stack_fold_psignw(x86_mmx %a0, x86_mmx %a1) { + ;CHECK-LABEL: stack_fold_psignw + ;CHECK: psignw {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload + %1 = tail call x86_mmx asm sideeffect "nop", "=y,~{mm2},~{mm3},~{mm4},~{mm5},~{mm6},~{mm7}"() + %2 = call x86_mmx @llvm.x86.ssse3.psign.w(x86_mmx %a0, x86_mmx %a1) nounwind readnone + ret x86_mmx %2 +} +declare x86_mmx @llvm.x86.ssse3.psign.w(x86_mmx, x86_mmx) nounwind readnone + define x86_mmx @stack_fold_pslld(x86_mmx %a, x86_mmx %b) { ;CHECK-LABEL: stack_fold_pslld ;CHECK: pslld {{-?[0-9]*}}(%rsp), {{%mm[0-7]}} {{.*#+}} 8-byte Folded Reload diff --git a/test/CodeGen/X86/stack-folding-x86_64.ll b/test/CodeGen/X86/stack-folding-x86_64.ll index 211227916a09..f96880d0237a 100644 --- a/test/CodeGen/X86/stack-folding-x86_64.ll +++ b/test/CodeGen/X86/stack-folding-x86_64.ll @@ -1,4 +1,4 @@ -; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mcpu=x86-64 < %s | FileCheck %s +; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown < %s | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-unknown" diff --git a/test/CodeGen/X86/stack-folding-xop.ll b/test/CodeGen/X86/stack-folding-xop.ll index 44a0d1dc6582..d0c48b400804 100644 --- a/test/CodeGen/X86/stack-folding-xop.ll +++ b/test/CodeGen/X86/stack-folding-xop.ll @@ -1,4 +1,4 @@ -; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx,+xop < %s | FileCheck %s +; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx,+xop < %s | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-unknown" diff --git a/test/CodeGen/X86/stack-probe-size.ll b/test/CodeGen/X86/stack-probe-size.ll index 21482c3abded..4d1f88d11172 100644 --- a/test/CodeGen/X86/stack-probe-size.ll +++ b/test/CodeGen/X86/stack-probe-size.ll @@ -6,10 +6,9 @@ ; stack probe size equals the page size (4096 bytes for all x86 targets), and ; this is unlikely to change in the future. ; -; RUN: llc < %s | FileCheck %s +; RUN: llc -mtriple=i686-windows-msvc < %s | FileCheck %s target datalayout = "e-m:w-p:32:32-i64:64-f80:32-n8:16:32-S32" -target triple = "i686-pc-windows-msvc" define i32 @test1() "stack-probe-size"="0" { %buffer = alloca [4095 x i8] diff --git a/test/CodeGen/X86/stack-protector-dbginfo.ll b/test/CodeGen/X86/stack-protector-dbginfo.ll index 3aba19464b9d..237b96603c00 100644 --- a/test/CodeGen/X86/stack-protector-dbginfo.ll +++ b/test/CodeGen/X86/stack-protector-dbginfo.ll @@ -8,7 +8,7 @@ @a = external global { i64, [56 x i8] }, align 32 ; Function Attrs: nounwind sspreq -define i32 @_Z18read_response_sizev() #0 { +define i32 @_Z18read_response_sizev() #0 !dbg !9 { entry: tail call void @llvm.dbg.value(metadata !22, i64 0, metadata !23, metadata !DIExpression()), !dbg !39 %0 = load i64, i64* getelementptr inbounds ({ i64, [56 x i8] }, { i64, [56 x i8] }* @a, i32 0, i32 0), align 8, !dbg !40 @@ -25,7 +25,7 @@ attributes #0 = { sspreq } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!21, !72} -!0 = !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !5, subprograms: !8, globals: !20, imports: !5) +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: true, emissionKind: 1, file: !1, enums: !2, retainedTypes: !5, subprograms: !8, globals: !20, imports: !5) !1 = !DIFile(filename: "", directory: "/Users/matt/ryan_bug") !2 = !{!3} !3 = !DICompositeType(tag: DW_TAG_enumeration_type, line: 20, size: 32, align: 32, file: !1, scope: !4, elements: !6) @@ -34,22 +34,22 @@ attributes #0 = { sspreq } !6 = !{!7} !7 = !DIEnumerator(name: "max_frame_size", value: 0) ; [ DW_TAG_enumerator ] [max_frame_size :: 0] !8 = !{!9, !24, !41, !65} -!9 = !DISubprogram(name: "read_response_size", linkageName: "_Z18read_response_sizev", line: 27, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 27, file: !1, scope: !10, type: !11, function: i32 ()* @_Z18read_response_sizev, variables: !14) +!9 = distinct !DISubprogram(name: "read_response_size", linkageName: "_Z18read_response_sizev", line: 27, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 27, file: !1, scope: !10, type: !11, variables: !14) !10 = !DIFile(filename: "", directory: "/Users/matt/ryan_bug") !11 = !DISubroutineType(types: !12) !12 = !{!13} !13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !14 = !{!15, !19} -!15 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "b", line: 28, scope: !9, file: !10, type: !16) +!15 = !DILocalVariable(name: "b", line: 28, scope: !9, file: !10, type: !16) !16 = !DICompositeType(tag: DW_TAG_structure_type, name: "B", line: 16, size: 32, align: 32, file: !1, elements: !17) !17 = !{!18} !18 = !DIDerivedType(tag: DW_TAG_member, name: "end_of_file", line: 17, size: 32, align: 32, file: !1, scope: !16, baseType: !13) -!19 = !DILocalVariable(tag: DW_TAG_auto_variable, name: "c", line: 29, scope: !9, file: !10, type: !13) +!19 = !DILocalVariable(name: "c", line: 29, scope: !9, file: !10, type: !13) !20 = !{} !21 = !{i32 2, !"Dwarf Version", i32 2} !22 = !{i64* getelementptr inbounds ({ i64, [56 x i8] }, { i64, [56 x i8] }* @a, i32 0, i32 0)} -!23 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p2", line: 12, arg: 2, scope: !24, file: !10, type: !32) -!24 = !DISubprogram(name: "min", linkageName: "_ZN3__13minIyEERKT_S3_RS1_", line: 12, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 12, file: !1, scope: !25, type: !27, templateParams: !33, variables: !35) +!23 = !DILocalVariable(name: "p2", line: 12, arg: 2, scope: !24, file: !10, type: !32) +!24 = distinct !DISubprogram(name: "min", linkageName: "_ZN3__13minIyEERKT_S3_RS1_", line: 12, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 12, file: !1, scope: !25, type: !27, templateParams: !33, variables: !35) !25 = !DINamespace(name: "__1", line: 1, file: !26, scope: null) !26 = !DIFile(filename: "main.cpp", directory: "/Users/matt/ryan_bug") !27 = !DISubroutineType(types: !28) @@ -61,12 +61,12 @@ attributes #0 = { sspreq } !33 = !{!34} !34 = !DITemplateTypeParameter(name: "_Tp", type: !31) !35 = !{!36, !37} -!36 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 12, arg: 1, scope: !24, file: !10, type: !29) -!37 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p2", line: 12, arg: 2, scope: !24, file: !10, type: !32) +!36 = !DILocalVariable(name: "p1", line: 12, arg: 1, scope: !24, file: !10, type: !29) +!37 = !DILocalVariable(name: "p2", line: 12, arg: 2, scope: !24, file: !10, type: !32) !38 = !DILocation(line: 33, scope: !9) !39 = !DILocation(line: 12, scope: !24, inlinedAt: !38) !40 = !DILocation(line: 9, scope: !41, inlinedAt: !59) -!41 = !DISubprogram(name: "min", linkageName: "_ZN3__13minIyNS_1AEEERKT_S4_RS2_T0_", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 8, file: !1, scope: !25, type: !42, templateParams: !53, variables: !55) +!41 = distinct !DISubprogram(name: "min", linkageName: "_ZN3__13minIyNS_1AEEERKT_S4_RS2_T0_", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 8, file: !1, scope: !25, type: !42, templateParams: !53, variables: !55) !42 = !DISubroutineType(types: !43) !43 = !{!29, !29, !32, !44} !44 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", size: 8, align: 8, file: !1, scope: !25, elements: !45) @@ -80,17 +80,17 @@ attributes #0 = { sspreq } !53 = !{!34, !54} !54 = !DITemplateTypeParameter(name: "_Compare", type: !44) !55 = !{!56, !57, !58} -!56 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 7, arg: 1, scope: !41, file: !10, type: !29) -!57 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p2", line: 7, arg: 2, scope: !41, file: !10, type: !32) -!58 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p3", line: 8, arg: 3, scope: !41, file: !10, type: !44) +!56 = !DILocalVariable(name: "p1", line: 7, arg: 1, scope: !41, file: !10, type: !29) +!57 = !DILocalVariable(name: "p2", line: 7, arg: 2, scope: !41, file: !10, type: !32) +!58 = !DILocalVariable(name: "p3", line: 8, arg: 3, scope: !41, file: !10, type: !44) !59 = !DILocation(line: 13, scope: !24, inlinedAt: !38) !63 = !{i32 undef} -!64 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 1, arg: 2, scope: !65, file: !10, type: !50) -!65 = !DISubprogram(name: "operator()", linkageName: "_ZN3__11AclERKiS2_", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !25, type: !47, declaration: !46, variables: !66) +!64 = !DILocalVariable(name: "p1", line: 1, arg: 2, scope: !65, file: !10, type: !50) +!65 = distinct !DISubprogram(name: "operator()", linkageName: "_ZN3__11AclERKiS2_", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, scopeLine: 2, file: !1, scope: !25, type: !47, declaration: !46, variables: !66) !66 = !{!67, !69, !70} -!67 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !65, type: !68) +!67 = !DILocalVariable(name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !65, type: !68) !68 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !44) -!69 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "p1", line: 1, arg: 2, scope: !65, file: !10, type: !50) -!70 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "", line: 2, arg: 3, scope: !65, file: !10, type: !50) +!69 = !DILocalVariable(name: "p1", line: 1, arg: 2, scope: !65, file: !10, type: !50) +!70 = !DILocalVariable(name: "", line: 2, arg: 3, scope: !65, file: !10, type: !50) !71 = !DILocation(line: 1, scope: !65, inlinedAt: !40) !72 = !{i32 1, !"Debug Info Version", i32 3} diff --git a/test/CodeGen/X86/stack-protector-weight.ll b/test/CodeGen/X86/stack-protector-weight.ll index 4220a4c46a0a..dea66d28e3dd 100644 --- a/test/CodeGen/X86/stack-protector-weight.ll +++ b/test/CodeGen/X86/stack-protector-weight.ll @@ -2,13 +2,13 @@ ; RUN: llc -mtriple=x86_64-apple-darwin -print-machineinstrs=expand-isel-pseudos -enable-selectiondag-sp=false %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=IR ; SELDAG: # Machine code for function test_branch_weights: -; SELDAG: Successors according to CFG: BB#[[SUCCESS:[0-9]+]](1048575) BB#[[FAILURE:[0-9]+]](1) +; SELDAG: Successors according to CFG: BB#[[SUCCESS:[0-9]+]]({{[0-9a-fx/= ]+}}100.00%) BB#[[FAILURE:[0-9]+]] ; SELDAG: BB#[[FAILURE]]: ; SELDAG: CALL64pcrel32 ; SELDAG: BB#[[SUCCESS]]: ; IR: # Machine code for function test_branch_weights: -; IR: Successors according to CFG: BB#[[SUCCESS:[0-9]+]](1048575) BB#[[FAILURE:[0-9]+]](1) +; IR: Successors according to CFG: BB#[[SUCCESS:[0-9]+]]({{[0-9a-fx/= ]+}}100.00%) BB#[[FAILURE:[0-9]+]] ; IR: BB#[[SUCCESS]]: ; IR: BB#[[FAILURE]]: ; IR: CALL64pcrel32 diff --git a/test/CodeGen/X86/stackmap-frame-setup.ll b/test/CodeGen/X86/stackmap-frame-setup.ll new file mode 100644 index 000000000000..076e2482f8ba --- /dev/null +++ b/test/CodeGen/X86/stackmap-frame-setup.ll @@ -0,0 +1,20 @@ +; RUN: llc -o /dev/null -verify-machineinstrs -mtriple=x86_64-apple-darwin -mcpu=corei7 -stop-after machine-sink %s | FileCheck %s --check-prefix=ISEL +; RUN: llc -o /dev/null -verify-machineinstrs -mtriple=x86_64-apple-darwin -mcpu=corei7 -fast-isel -fast-isel-abort=1 -stop-after machine-sink %s | FileCheck %s --check-prefix=FAST-ISEL + +define void @caller_meta_leaf() { +entry: + %metadata = alloca i64, i32 3, align 8 + store i64 11, i64* %metadata + store i64 12, i64* %metadata + store i64 13, i64* %metadata +; ISEL: ADJCALLSTACKDOWN64 0, 0, implicit-def +; ISEL-NEXT: STACKMAP +; ISEL-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def + call void (i64, i32, ...) @llvm.experimental.stackmap(i64 4, i32 0, i64* %metadata) +; FAST-ISEL: ADJCALLSTACKDOWN64 0, 0, implicit-def +; FAST-ISEL-NEXT: STACKMAP +; FAST-ISEL-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def + ret void +} + +declare void @llvm.experimental.stackmap(i64, i32, ...) diff --git a/test/CodeGen/X86/statepoint-allocas.ll b/test/CodeGen/X86/statepoint-allocas.ll index 4af33e1f5478..fa2621e7d2fe 100644 --- a/test/CodeGen/X86/statepoint-allocas.ll +++ b/test/CodeGen/X86/statepoint-allocas.ll @@ -16,12 +16,12 @@ define i32 addrspace(1)* @test(i32 addrspace(1)* %ptr) gc "statepoint-example" { ; CHECK: movq %rdi, (%rsp) ; CHECK: callq return_i1 ; CHECK: movq (%rsp), %rax -; CHECK: popq %rdx +; CHECK: popq %rcx ; CHECK: retq entry: %alloca = alloca i32 addrspace(1)*, align 8 store i32 addrspace(1)* %ptr, i32 addrspace(1)** %alloca - call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)** %alloca) + call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)** %alloca) %rel = load i32 addrspace(1)*, i32 addrspace(1)** %alloca ret i32 addrspace(1)* %rel } @@ -33,16 +33,16 @@ define i32 addrspace(1)* @test2(i32 addrspace(1)* %ptr) gc "statepoint-example" ; CHECK: movq %rdi, (%rsp) ; CHECK: callq return_i1 ; CHECK: xorl %eax, %eax -; CHECK: popq %rdx +; CHECK: popq %rcx ; CHECK: retq entry: %alloca = alloca i32 addrspace(1)*, align 8 store i32 addrspace(1)* %ptr, i32 addrspace(1)** %alloca - call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 1, i32 addrspace(1)** %alloca) + call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 1, i32 addrspace(1)** %alloca) ret i32 addrspace(1)* null } -declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...) +declare token @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...) ; CHECK-LABEL: .section .llvm_stackmaps diff --git a/test/CodeGen/X86/statepoint-call-lowering.ll b/test/CodeGen/X86/statepoint-call-lowering.ll index 8f352b7728c3..a8fa3cb37782 100644 --- a/test/CodeGen/X86/statepoint-call-lowering.ll +++ b/test/CodeGen/X86/statepoint-call-lowering.ll @@ -5,10 +5,13 @@ target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-pc-linux-gnu" +%struct = type { i64, i64 } + declare zeroext i1 @return_i1() declare zeroext i32 @return_i32() declare i32* @return_i32ptr() declare float @return_float() +declare %struct @return_struct() declare void @varargf(i32, ...) define i1 @test_i1_return() gc "statepoint-example" { @@ -17,11 +20,11 @@ define i1 @test_i1_return() gc "statepoint-example" { ; state arguments to the statepoint ; CHECK: pushq %rax ; CHECK: callq return_i1 -; CHECK: popq %rdx +; CHECK: popq %rcx ; CHECK: retq entry: - %safepoint_token = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) - %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(i32 %safepoint_token) + %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) + %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token) ret i1 %call1 } @@ -29,11 +32,11 @@ define i32 @test_i32_return() gc "statepoint-example" { ; CHECK-LABEL: test_i32_return ; CHECK: pushq %rax ; CHECK: callq return_i32 -; CHECK: popq %rdx +; CHECK: popq %rcx ; CHECK: retq entry: - %safepoint_token = tail call i32 (i64, i32, i32 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32f(i64 0, i32 0, i32 ()* @return_i32, i32 0, i32 0, i32 0, i32 0) - %call1 = call zeroext i32 @llvm.experimental.gc.result.i32(i32 %safepoint_token) + %safepoint_token = tail call token (i64, i32, i32 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32f(i64 0, i32 0, i32 ()* @return_i32, i32 0, i32 0, i32 0, i32 0) + %call1 = call zeroext i32 @llvm.experimental.gc.result.i32(token %safepoint_token) ret i32 %call1 } @@ -41,11 +44,11 @@ define i32* @test_i32ptr_return() gc "statepoint-example" { ; CHECK-LABEL: test_i32ptr_return ; CHECK: pushq %rax ; CHECK: callq return_i32ptr -; CHECK: popq %rdx +; CHECK: popq %rcx ; CHECK: retq entry: - %safepoint_token = tail call i32 (i64, i32, i32* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p0i32f(i64 0, i32 0, i32* ()* @return_i32ptr, i32 0, i32 0, i32 0, i32 0) - %call1 = call i32* @llvm.experimental.gc.result.p0i32(i32 %safepoint_token) + %safepoint_token = tail call token (i64, i32, i32* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p0i32f(i64 0, i32 0, i32* ()* @return_i32ptr, i32 0, i32 0, i32 0, i32 0) + %call1 = call i32* @llvm.experimental.gc.result.p0i32(token %safepoint_token) ret i32* %call1 } @@ -56,23 +59,35 @@ define float @test_float_return() gc "statepoint-example" { ; CHECK: popq %rax ; CHECK: retq entry: - %safepoint_token = tail call i32 (i64, i32, float ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_f32f(i64 0, i32 0, float ()* @return_float, i32 0, i32 0, i32 0, i32 0) - %call1 = call float @llvm.experimental.gc.result.f32(i32 %safepoint_token) + %safepoint_token = tail call token (i64, i32, float ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_f32f(i64 0, i32 0, float ()* @return_float, i32 0, i32 0, i32 0, i32 0) + %call1 = call float @llvm.experimental.gc.result.f32(token %safepoint_token) ret float %call1 } +define %struct @test_struct_return() gc "statepoint-example" { +; CHECK-LABEL: test_struct_return +; CHECK: pushq %rax +; CHECK: callq return_struct +; CHECK: popq %rcx +; CHECK: retq +entry: + %safepoint_token = tail call token (i64, i32, %struct ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_structf(i64 0, i32 0, %struct ()* @return_struct, i32 0, i32 0, i32 0, i32 0) + %call1 = call %struct @llvm.experimental.gc.result.struct(token %safepoint_token) + ret %struct %call1 +} + define i1 @test_relocate(i32 addrspace(1)* %a) gc "statepoint-example" { ; CHECK-LABEL: test_relocate ; Check that an ununsed relocate has no code-generation impact ; CHECK: pushq %rax ; CHECK: callq return_i1 -; CHECK-NEXT: .Ltmp9: -; CHECK-NEXT: popq %rdx +; CHECK-NEXT: .Ltmp11: +; CHECK-NEXT: popq %rcx ; CHECK-NEXT: retq entry: - %safepoint_token = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %a) - %call1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 7, i32 7) - %call2 = call zeroext i1 @llvm.experimental.gc.result.i1(i32 %safepoint_token) + %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %a) + %call1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 7, i32 7) + %call2 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token) ret i1 %call2 } @@ -81,7 +96,7 @@ define void @test_void_vararg() gc "statepoint-example" { ; Check a statepoint wrapping a *void* returning vararg function works ; CHECK: callq varargf entry: - %safepoint_token = tail call i32 (i64, i32, void (i32, ...)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidi32varargf(i64 0, i32 0, void (i32, ...)* @varargf, i32 2, i32 0, i32 42, i32 43, i32 0, i32 0) + %safepoint_token = tail call token (i64, i32, void (i32, ...)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidi32varargf(i64 0, i32 0, void (i32, ...)* @varargf, i32 2, i32 0, i32 42, i32 43, i32 0, i32 0) ;; if we try to use the result from a statepoint wrapping a ;; non-void-returning varargf, we will experience a crash. ret void @@ -92,26 +107,54 @@ define i1 @test_i1_return_patchable() gc "statepoint-example" { ; A patchable variant of test_i1_return ; CHECK: pushq %rax ; CHECK: nopl -; CHECK: popq %rdx +; CHECK: popq %rcx ; CHECK: retq entry: - %safepoint_token = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 3, i1 ()*null, i32 0, i32 0, i32 0, i32 0) - %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(i32 %safepoint_token) + %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 3, i1 ()*null, i32 0, i32 0, i32 0, i32 0) + %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token) ret i1 %call1 } -declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...) -declare i1 @llvm.experimental.gc.result.i1(i32) +declare void @consume(i32 addrspace(1)* %obj) + +define i1 @test_cross_bb(i32 addrspace(1)* %a, i1 %external_cond) gc "statepoint-example" { +; CHECK-LABEL: test_cross_bb +; CHECK: movq +; CHECK: callq return_i1 +; CHECK: %left +; CHECK: movq +; CHECK-NEXT: callq consume +; CHECK: retq +entry: + %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %a) + br i1 %external_cond, label %left, label %right + +left: + %call1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 7, i32 7) + %call2 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token) + call void @consume(i32 addrspace(1)* %call1) + ret i1 %call2 + +right: + ret i1 true +} + + +declare token @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...) +declare i1 @llvm.experimental.gc.result.i1(token) + +declare token @llvm.experimental.gc.statepoint.p0f_i32f(i64, i32, i32 ()*, i32, i32, ...) +declare i32 @llvm.experimental.gc.result.i32(token) -declare i32 @llvm.experimental.gc.statepoint.p0f_i32f(i64, i32, i32 ()*, i32, i32, ...) -declare i32 @llvm.experimental.gc.result.i32(i32) +declare token @llvm.experimental.gc.statepoint.p0f_p0i32f(i64, i32, i32* ()*, i32, i32, ...) +declare i32* @llvm.experimental.gc.result.p0i32(token) -declare i32 @llvm.experimental.gc.statepoint.p0f_p0i32f(i64, i32, i32* ()*, i32, i32, ...) -declare i32* @llvm.experimental.gc.result.p0i32(i32) +declare token @llvm.experimental.gc.statepoint.p0f_f32f(i64, i32, float ()*, i32, i32, ...) +declare float @llvm.experimental.gc.result.f32(token) -declare i32 @llvm.experimental.gc.statepoint.p0f_f32f(i64, i32, float ()*, i32, i32, ...) -declare float @llvm.experimental.gc.result.f32(i32) +declare token @llvm.experimental.gc.statepoint.p0f_structf(i64, i32, %struct ()*, i32, i32, ...) +declare %struct @llvm.experimental.gc.result.struct(token) -declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidi32varargf(i64, i32, void (i32, ...)*, i32, i32, ...) +declare token @llvm.experimental.gc.statepoint.p0f_isVoidi32varargf(i64, i32, void (i32, ...)*, i32, i32, ...) -declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) +declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32, i32) diff --git a/test/CodeGen/X86/statepoint-far-call.ll b/test/CodeGen/X86/statepoint-far-call.ll index cd8dd0f35a20..2ebf38c5c019 100644 --- a/test/CodeGen/X86/statepoint-far-call.ll +++ b/test/CodeGen/X86/statepoint-far-call.ll @@ -14,9 +14,9 @@ define void @test_far_call() gc "statepoint-example" { ; CHECK: retq entry: - %safepoint_token = call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* inttoptr (i64 140727162896504 to void ()*), i32 0, i32 0, i32 0, i32 0) + %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* inttoptr (i64 140727162896504 to void ()*), i32 0, i32 0, i32 0, i32 0) ret void } -declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...) +declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...) diff --git a/test/CodeGen/X86/statepoint-forward.ll b/test/CodeGen/X86/statepoint-forward.ll index 698229e705f4..d97bc0c75602 100644 --- a/test/CodeGen/X86/statepoint-forward.ll +++ b/test/CodeGen/X86/statepoint-forward.ll @@ -25,8 +25,8 @@ entry: %before = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(1)* %p %cmp1 = call i1 @f(i32 addrspace(1)* %before) call void @llvm.assume(i1 %cmp1) - %safepoint_token = tail call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* addrspace(1)* %p) - %pnew = call i32 addrspace(1)* addrspace(1)* @llvm.experimental.gc.relocate.p1p1i32(i32 %safepoint_token, i32 7, i32 7) + %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* addrspace(1)* %p) + %pnew = call i32 addrspace(1)* addrspace(1)* @llvm.experimental.gc.relocate.p1p1i32(token %safepoint_token, i32 7, i32 7) %after = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(1)* %pnew %cmp2 = call i1 @f(i32 addrspace(1)* %after) ret i1 %cmp2 @@ -44,8 +44,8 @@ entry: %cmp1 = call i1 @f(i32 addrspace(1)* %v) call void @llvm.assume(i1 %cmp1) store i32 addrspace(1)* %v, i32 addrspace(1)* addrspace(1)* %p - %safepoint_token = tail call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* addrspace(1)* %p) - %pnew = call i32 addrspace(1)* addrspace(1)* @llvm.experimental.gc.relocate.p1p1i32(i32 %safepoint_token, i32 7, i32 7) + %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* addrspace(1)* %p) + %pnew = call i32 addrspace(1)* addrspace(1)* @llvm.experimental.gc.relocate.p1p1i32(token %safepoint_token, i32 7, i32 7) %after = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(1)* %pnew %cmp2 = call i1 @f(i32 addrspace(1)* %after) ret i1 %cmp2 @@ -72,7 +72,7 @@ entry: %before = load i32 addrspace(1)*, i32 addrspace(1)** %p %cmp1 = call i1 @f(i32 addrspace(1)* %before) call void @llvm.assume(i1 %cmp1) - call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) + call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) %after = load i32 addrspace(1)*, i32 addrspace(1)** %p %cmp2 = call i1 @f(i32 addrspace(1)* %after) ret i1 %cmp2 @@ -90,7 +90,7 @@ entry: %cmp1 = call i1 @f(i32 addrspace(1)* %v) call void @llvm.assume(i1 %cmp1) store i32 addrspace(1)* %v, i32 addrspace(1)** %p - call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) + call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) %after = load i32 addrspace(1)*, i32 addrspace(1)** %p %cmp2 = call i1 @f(i32 addrspace(1)* %after) ret i1 %cmp2 @@ -102,5 +102,5 @@ entry: } declare void @llvm.assume(i1) -declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...) -declare i32 addrspace(1)* addrspace(1)* @llvm.experimental.gc.relocate.p1p1i32(i32, i32, i32) #3 +declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...) +declare i32 addrspace(1)* addrspace(1)* @llvm.experimental.gc.relocate.p1p1i32(token, i32, i32) #3 diff --git a/test/CodeGen/X86/statepoint-gctransition-call-lowering.ll b/test/CodeGen/X86/statepoint-gctransition-call-lowering.ll index 61b8ded2c472..b4ba0964fdd6 100644 --- a/test/CodeGen/X86/statepoint-gctransition-call-lowering.ll +++ b/test/CodeGen/X86/statepoint-gctransition-call-lowering.ll @@ -18,11 +18,11 @@ define i1 @test_i1_return() gc "statepoint-example" { ; state arguments to the statepoint ; CHECK: pushq %rax ; CHECK: callq return_i1 -; CHECK: popq %rdx +; CHECK: popq %rcx ; CHECK: retq entry: - %safepoint_token = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 1, i32 0, i32 0) - %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(i32 %safepoint_token) + %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 1, i32 0, i32 0) + %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token) ret i1 %call1 } @@ -30,11 +30,11 @@ define i32 @test_i32_return() gc "statepoint-example" { ; CHECK-LABEL: test_i32_return ; CHECK: pushq %rax ; CHECK: callq return_i32 -; CHECK: popq %rdx +; CHECK: popq %rcx ; CHECK: retq entry: - %safepoint_token = tail call i32 (i64, i32, i32 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32f(i64 0, i32 0, i32 ()* @return_i32, i32 0, i32 1, i32 0, i32 0) - %call1 = call zeroext i32 @llvm.experimental.gc.result.i32(i32 %safepoint_token) + %safepoint_token = tail call token (i64, i32, i32 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32f(i64 0, i32 0, i32 ()* @return_i32, i32 0, i32 1, i32 0, i32 0) + %call1 = call zeroext i32 @llvm.experimental.gc.result.i32(token %safepoint_token) ret i32 %call1 } @@ -42,11 +42,11 @@ define i32* @test_i32ptr_return() gc "statepoint-example" { ; CHECK-LABEL: test_i32ptr_return ; CHECK: pushq %rax ; CHECK: callq return_i32ptr -; CHECK: popq %rdx +; CHECK: popq %rcx ; CHECK: retq entry: - %safepoint_token = tail call i32 (i64, i32, i32* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p0i32f(i64 0, i32 0, i32* ()* @return_i32ptr, i32 0, i32 1, i32 0, i32 0) - %call1 = call i32* @llvm.experimental.gc.result.p0i32(i32 %safepoint_token) + %safepoint_token = tail call token (i64, i32, i32* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p0i32f(i64 0, i32 0, i32* ()* @return_i32ptr, i32 0, i32 1, i32 0, i32 0) + %call1 = call i32* @llvm.experimental.gc.result.p0i32(token %safepoint_token) ret i32* %call1 } @@ -57,8 +57,8 @@ define float @test_float_return() gc "statepoint-example" { ; CHECK: popq %rax ; CHECK: retq entry: - %safepoint_token = tail call i32 (i64, i32, float ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_f32f(i64 0, i32 0, float ()* @return_float, i32 0, i32 1, i32 0, i32 0) - %call1 = call float @llvm.experimental.gc.result.f32(i32 %safepoint_token) + %safepoint_token = tail call token (i64, i32, float ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_f32f(i64 0, i32 0, float ()* @return_float, i32 0, i32 1, i32 0, i32 0) + %call1 = call float @llvm.experimental.gc.result.f32(token %safepoint_token) ret float %call1 } @@ -68,12 +68,12 @@ define i1 @test_relocate(i32 addrspace(1)* %a) gc "statepoint-example" { ; CHECK: pushq %rax ; CHECK: callq return_i1 ; CHECK-NEXT: .Ltmp9: -; CHECK-NEXT: popq %rdx +; CHECK-NEXT: popq %rcx ; CHECK-NEXT: retq entry: - %safepoint_token = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 1, i32 0, i32 0, i32 addrspace(1)* %a) - %call1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 7, i32 7) - %call2 = call zeroext i1 @llvm.experimental.gc.result.i1(i32 %safepoint_token) + %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 1, i32 0, i32 0, i32 addrspace(1)* %a) + %call1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 7, i32 7) + %call2 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token) ret i1 %call2 } @@ -82,7 +82,7 @@ define void @test_void_vararg() gc "statepoint-example" { ; Check a statepoint wrapping a *void* returning vararg function works ; CHECK: callq varargf entry: - %safepoint_token = tail call i32 (i64, i32, void (i32, ...)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidi32varargf(i64 0, i32 0, void (i32, ...)* @varargf, i32 2, i32 1, i32 42, i32 43, i32 0, i32 0) + %safepoint_token = tail call token (i64, i32, void (i32, ...)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidi32varargf(i64 0, i32 0, void (i32, ...)* @varargf, i32 2, i32 1, i32 42, i32 43, i32 0, i32 0) ;; if we try to use the result from a statepoint wrapping a ;; non-void-returning varargf, we will experience a crash. ret void @@ -92,12 +92,12 @@ define i32 @test_transition_args() gc "statepoint-example" { ; CHECK-LABEL: test_transition_args ; CHECK: pushq %rax ; CHECK: callq return_i32 -; CHECK: popq %rdx +; CHECK: popq %rcx ; CHECK: retq entry: %val = alloca i32 - %safepoint_token = call i32 (i64, i32, i32 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32f(i64 0, i32 0, i32 ()* @return_i32, i32 0, i32 1, i32 2, i32* %val, i64 42, i32 0) - %call1 = call i32 @llvm.experimental.gc.result.i32(i32 %safepoint_token) + %safepoint_token = call token (i64, i32, i32 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32f(i64 0, i32 0, i32 ()* @return_i32, i32 0, i32 1, i32 2, i32* %val, i64 42, i32 0) + %call1 = call i32 @llvm.experimental.gc.result.i32(token %safepoint_token) ret i32 %call1 } @@ -105,29 +105,29 @@ define i32 @test_transition_args_2() gc "statepoint-example" { ; CHECK-LABEL: test_transition_args_2 ; CHECK: pushq %rax ; CHECK: callq return_i32 -; CHECK: popq %rdx +; CHECK: popq %rcx ; CHECK: retq entry: %val = alloca i32 %arg = alloca i8 - %safepoint_token = call i32 (i64, i32, i32 (i32, i8*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32i32p0i8f(i64 0, i32 0, i32 (i32, i8*)* @return_i32_with_args, i32 2, i32 1, i32 0, i8* %arg, i32 2, i32* %val, i64 42, i32 0) - %call1 = call i32 @llvm.experimental.gc.result.i32(i32 %safepoint_token) + %safepoint_token = call token (i64, i32, i32 (i32, i8*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32i32p0i8f(i64 0, i32 0, i32 (i32, i8*)* @return_i32_with_args, i32 2, i32 1, i32 0, i8* %arg, i32 2, i32* %val, i64 42, i32 0) + %call1 = call i32 @llvm.experimental.gc.result.i32(token %safepoint_token) ret i32 %call1 } -declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...) -declare i1 @llvm.experimental.gc.result.i1(i32) +declare token @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...) +declare i1 @llvm.experimental.gc.result.i1(token) -declare i32 @llvm.experimental.gc.statepoint.p0f_i32f(i64, i32, i32 ()*, i32, i32, ...) -declare i32 @llvm.experimental.gc.statepoint.p0f_i32i32p0i8f(i64, i32, i32 (i32, i8*)*, i32, i32, ...) -declare i32 @llvm.experimental.gc.result.i32(i32) +declare token @llvm.experimental.gc.statepoint.p0f_i32f(i64, i32, i32 ()*, i32, i32, ...) +declare token @llvm.experimental.gc.statepoint.p0f_i32i32p0i8f(i64, i32, i32 (i32, i8*)*, i32, i32, ...) +declare i32 @llvm.experimental.gc.result.i32(token) -declare i32 @llvm.experimental.gc.statepoint.p0f_p0i32f(i64, i32, i32* ()*, i32, i32, ...) -declare i32* @llvm.experimental.gc.result.p0i32(i32) +declare token @llvm.experimental.gc.statepoint.p0f_p0i32f(i64, i32, i32* ()*, i32, i32, ...) +declare i32* @llvm.experimental.gc.result.p0i32(token) -declare i32 @llvm.experimental.gc.statepoint.p0f_f32f(i64, i32, float ()*, i32, i32, ...) -declare float @llvm.experimental.gc.result.f32(i32) +declare token @llvm.experimental.gc.statepoint.p0f_f32f(i64, i32, float ()*, i32, i32, ...) +declare float @llvm.experimental.gc.result.f32(token) -declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidi32varargf(i64, i32, void (i32, ...)*, i32, i32, ...) +declare token @llvm.experimental.gc.statepoint.p0f_isVoidi32varargf(i64, i32, void (i32, ...)*, i32, i32, ...) -declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) \ No newline at end of file +declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32, i32) \ No newline at end of file diff --git a/test/CodeGen/X86/statepoint-invoke.ll b/test/CodeGen/X86/statepoint-invoke.ll index 81b9ab89ebca..1d38b2facc73 100644 --- a/test/CodeGen/X86/statepoint-invoke.ll +++ b/test/CodeGen/X86/statepoint-invoke.ll @@ -14,13 +14,13 @@ entry: ; CHECK: Ltmp{{[0-9]+}}: ; CHECK: callq some_call ; CHECK: Ltmp{{[0-9]+}}: - %0 = invoke i32 (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) + %0 = invoke token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) to label %invoke_safepoint_normal_dest unwind label %exceptional_return invoke_safepoint_normal_dest: ; CHECK: movq - %obj.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %0, i32 13, i32 13) - %obj1.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %0, i32 14, i32 14) + %obj.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %0, i32 13, i32 13) + %obj1.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %0, i32 14, i32 14) br label %normal_return normal_return: @@ -31,11 +31,10 @@ exceptional_return: ; CHECK: Ltmp{{[0-9]+}}: ; CHECK: movq ; CHECK: retq - %landing_pad = landingpad { i8*, i32 } + %landing_pad = landingpad token cleanup - %relocate_token = extractvalue { i8*, i32 } %landing_pad, 1 - %obj.relocated1 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 13, i32 13) - %obj1.relocated1 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 14, i32 14) + %obj.relocated1 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 13, i32 13) + %obj1.relocated1 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 14, i32 14) ret i64 addrspace(1)* %obj1.relocated1 } ; CHECK-LABEL: GCC_except_table{{[0-9]+}}: @@ -51,22 +50,21 @@ entry: ; CHECK: .Ltmp{{[0-9]+}}: ; CHECK: callq some_other_call ; CHECK: .Ltmp{{[0-9]+}}: - %0 = invoke i32 (i64, i32, i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 0, i32 0, i64 addrspace(1)* (i64 addrspace(1)*)* @some_other_call, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) + %0 = invoke token (i64, i32, i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 0, i32 0, i64 addrspace(1)* (i64 addrspace(1)*)* @some_other_call, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) to label %normal_return unwind label %exceptional_return normal_return: ; CHECK: popq ; CHECK: retq - %ret_val = call i64 addrspace(1)* @llvm.experimental.gc.result.p1i64(i32 %0) + %ret_val = call i64 addrspace(1)* @llvm.experimental.gc.result.p1i64(token %0) ret i64 addrspace(1)* %ret_val exceptional_return: ; CHECK: .Ltmp{{[0-9]+}}: ; CHECK: movq - %landing_pad = landingpad { i8*, i32 } + %landing_pad = landingpad token cleanup - %relocate_token = extractvalue { i8*, i32 } %landing_pad, 1 - %obj.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 13, i32 13) + %obj.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 13, i32 13) ret i64 addrspace(1)* %obj.relocated } ; CHECK-LABEL: GCC_except_table{{[0-9]+}}: @@ -85,14 +83,14 @@ left: ; CHECK: movq %rdx, 8(%rsp) ; CHECK: movq ; CHECK: callq some_call - %sp1 = invoke i32 (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %val1, i64 addrspace(1)* %val2) + %sp1 = invoke token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %val1, i64 addrspace(1)* %val2) to label %left.relocs unwind label %exceptional_return.left left.relocs: ; CHECK: movq (%rsp), ; CHECK: movq 8(%rsp), [[REGVAL2:%[a-z]+]] - %val1.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %sp1, i32 13, i32 13) - %val2.relocated_left = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %sp1, i32 14, i32 14) + %val1.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp1, i32 13, i32 13) + %val2.relocated_left = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp1, i32 14, i32 14) br label %normal_return right: @@ -100,37 +98,35 @@ right: ; CHECK: movq ; CHECK: movq %rdx, (%rsp) ; CHECK: callq some_call - %sp2 = invoke i32 (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %val2, i64 addrspace(1)* %val3) + %sp2 = invoke token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %val2, i64 addrspace(1)* %val3) to label %right.relocs unwind label %exceptional_return.right right.relocs: ; CHECK: movq (%rsp), [[REGVAL2]] ; CHECK: movq - %val2.relocated_right = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %sp2, i32 13, i32 13) - %val3.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %sp2, i32 14, i32 14) + %val2.relocated_right = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp2, i32 13, i32 13) + %val3.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp2, i32 14, i32 14) br label %normal_return normal_return: ; CHECK-LABEL: %normal_return ; CHECK: cmoveq {{.*}}[[REGVAL2]]{{.*}} - ; CHECK retq + ; CHECK: retq %a1 = phi i64 addrspace(1)* [%val1.relocated, %left.relocs], [%val3.relocated, %right.relocs] %a2 = phi i64 addrspace(1)* [%val2.relocated_left, %left.relocs], [%val2.relocated_right, %right.relocs] %ret = select i1 %cond, i64 addrspace(1)* %a1, i64 addrspace(1)* %a2 ret i64 addrspace(1)* %ret exceptional_return.left: - %landing_pad = landingpad { i8*, i32 } + %landing_pad = landingpad token cleanup - %relocate_token = extractvalue { i8*, i32 } %landing_pad, 1 - %val.relocated2 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 13, i32 13) + %val.relocated2 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 13, i32 13) ret i64 addrspace(1)* %val.relocated2 exceptional_return.right: - %landing_pad1 = landingpad { i8*, i32 } + %landing_pad1 = landingpad token cleanup - %relocate_token1 = extractvalue { i8*, i32 } %landing_pad1, 1 - %val.relocated3 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token1, i32 13, i32 13) + %val.relocated3 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad1, i32 13, i32 13) ret i64 addrspace(1)* %val.relocated3 } @@ -139,7 +135,7 @@ define i64 addrspace(1)* @test_null_undef(i64 addrspace(1)* %val1) ; CHECK-LABEL: test_null_undef: entry: ; CHECK: callq some_call - %sp1 = invoke i32 (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* null, i64 addrspace(1)* undef) + %sp1 = invoke token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* null, i64 addrspace(1)* undef) to label %normal_return unwind label %exceptional_return normal_return: @@ -147,16 +143,15 @@ normal_return: ; CHECK: xorl %eax, %eax ; CHECK-NEXT: popq ; CHECK-NEXT: retq - %null.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %sp1, i32 13, i32 13) - %undef.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %sp1, i32 14, i32 14) + %null.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp1, i32 13, i32 13) + %undef.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp1, i32 14, i32 14) ret i64 addrspace(1)* %null.relocated exceptional_return: - %landing_pad = landingpad { i8*, i32 } + %landing_pad = landingpad token cleanup - %relocate_token = extractvalue { i8*, i32 } %landing_pad, 1 - %null.relocated2 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 13, i32 13) - %undef.relocated2 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 14, i32 14) + %null.relocated2 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 13, i32 13) + %undef.relocated2 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 14, i32 14) ret i64 addrspace(1)* %null.relocated2 } @@ -168,14 +163,14 @@ entry: %aa = addrspacecast i32* %a to i32 addrspace(1)* %c = inttoptr i64 15 to i64 addrspace(1)* ; CHECK: callq - %sp = invoke i32 (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %aa, i64 addrspace(1)* %c) + %sp = invoke token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %aa, i64 addrspace(1)* %c) to label %normal_return unwind label %exceptional_return normal_return: ; CHECK: leaq ; CHECK-NEXT: popq ; CHECK-NEXT: retq - %aa.rel = call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %sp, i32 13, i32 13) + %aa.rel = call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %sp, i32 13, i32 13) %aa.converted = bitcast i32 addrspace(1)* %aa.rel to i64 addrspace(1)* ret i64 addrspace(1)* %aa.converted @@ -183,16 +178,15 @@ exceptional_return: ; CHECK: movl $15 ; CHECK-NEXT: popq ; CHECK-NEXT: retq - %landing_pad = landingpad { i8*, i32 } + %landing_pad = landingpad token cleanup - %relocate_token = extractvalue { i8*, i32 } %landing_pad, 1 - %aa.rel2 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32 %relocate_token, i32 14, i32 14) + %aa.rel2 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 14, i32 14) ret i64 addrspace(1)* %aa.rel2 } -declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) -declare i32 @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64, i32, i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...) +declare token @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) +declare token @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64, i32, i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...) -declare i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(i32, i32, i32) -declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) -declare i64 addrspace(1)* @llvm.experimental.gc.result.p1i64(i32) +declare i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token, i32, i32) +declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32, i32) +declare i64 addrspace(1)* @llvm.experimental.gc.result.p1i64(token) diff --git a/test/CodeGen/X86/statepoint-stack-usage.ll b/test/CodeGen/X86/statepoint-stack-usage.ll index a4aa747af8cf..d4784212810f 100644 --- a/test/CodeGen/X86/statepoint-stack-usage.ll +++ b/test/CodeGen/X86/statepoint-stack-usage.ll @@ -16,17 +16,17 @@ define i32 @back_to_back_calls(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 a ; CHECK: movq %rsi, (%rsp) ; There should be no more than three moves ; CHECK-NOT: movq - %safepoint_token = tail call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) - %a1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 12) - %b1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 13) - %c1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 14) + %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) + %a1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 12, i32 12) + %b1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 12, i32 13) + %c1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 12, i32 14) ; CHECK: callq ; This is the key check. There should NOT be any memory moves here ; CHECK-NOT: movq - %safepoint_token2 = tail call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %c1, i32 addrspace(1)* %b1, i32 addrspace(1)* %a1) - %a2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 12, i32 14) - %b2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 12, i32 13) - %c2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 12, i32 12) + %safepoint_token2 = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %c1, i32 addrspace(1)* %b1, i32 addrspace(1)* %a1) + %a2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 12, i32 14) + %b2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 12, i32 13) + %c2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 12, i32 12) ; CHECK: callq ret i32 1 } @@ -39,17 +39,17 @@ define i32 @reserve_first(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrsp ; CHECK: movq %rdi, 16(%rsp) ; CHECK: movq %rdx, 8(%rsp) ; CHECK: movq %rsi, (%rsp) - %safepoint_token = tail call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) - %a1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 12) - %b1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 13) - %c1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 14) + %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) + %a1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 12, i32 12) + %b1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 12, i32 13) + %c1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 12, i32 14) ; CHECK: callq ; This is the key check. There should NOT be any memory moves here ; CHECK-NOT: movq - %safepoint_token2 = tail call i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 addrspace(1)* %a1, i32 0, i32 addrspace(1)* %c1, i32 0, i32 0, i32 addrspace(1)* %c1, i32 addrspace(1)* %b1, i32 addrspace(1)* %a1) - %a2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 12, i32 14) - %b2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 12, i32 13) - %c2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 12, i32 12) + %safepoint_token2 = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 addrspace(1)* %a1, i32 0, i32 addrspace(1)* %c1, i32 0, i32 0, i32 addrspace(1)* %c1, i32 addrspace(1)* %b1, i32 addrspace(1)* %a1) + %a2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 12, i32 14) + %b2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 12, i32 13) + %c2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 12, i32 12) ; CHECK: callq ret i32 1 } @@ -63,25 +63,25 @@ entry: ; CHECK: movq %rdx, 8(%rsp) ; CHECK: movq %rsi, (%rsp) ; CHECK: callq - %safepoint_token = invoke i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) + %safepoint_token = invoke token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) to label %normal_return unwind label %exceptional_return normal_return: - %a1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 12) - %b1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 13) - %c1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 12, i32 14) + %a1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 12, i32 12) + %b1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 12, i32 13) + %c1 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 12, i32 14) ; Should work even through bitcasts %c1.casted = bitcast i32 addrspace(1)* %c1 to i8 addrspace(1)* ; This is the key check. There should NOT be any memory moves here ; CHECK-NOT: movq ; CHECK: callq - %safepoint_token2 = invoke i32 (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i8 addrspace(1)* %c1.casted, i32 addrspace(1)* %b1, i32 addrspace(1)* %a1) + %safepoint_token2 = invoke token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* undef, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i8 addrspace(1)* %c1.casted, i32 addrspace(1)* %b1, i32 addrspace(1)* %a1) to label %normal_return2 unwind label %exceptional_return2 normal_return2: - %a2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 12, i32 14) - %b2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token2, i32 12, i32 13) - %c2 = tail call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32 %safepoint_token2, i32 12, i32 12) + %a2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 12, i32 14) + %b2 = tail call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 12, i32 13) + %c2 = tail call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %safepoint_token2, i32 12, i32 12) ret i32 1 exceptional_return: @@ -96,10 +96,10 @@ exceptional_return2: } ; Function Attrs: nounwind -declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) #3 -declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(i32, i32, i32) #3 +declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32, i32) #3 +declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32, i32) #3 -declare i32 @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...) +declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...) declare i32 @"personality_function"() diff --git a/test/CodeGen/X86/statepoint-stackmap-format.ll b/test/CodeGen/X86/statepoint-stackmap-format.ll index e18476cee53c..4f8b2ce6efd9 100644 --- a/test/CodeGen/X86/statepoint-stackmap-format.ll +++ b/test/CodeGen/X86/statepoint-stackmap-format.ll @@ -1,5 +1,5 @@ ; RUN: llc < %s -mtriple="x86_64-pc-linux-gnu" | FileCheck %s -; RUN: llc < %s -mtriple="x86_64-pc-win64-coff" | FileCheck %s +; RUN: llc < %s -mtriple="x86_64-pc-unknown-elf" | FileCheck %s ; This test is a sanity check to ensure statepoints are generating StackMap ; sections correctly. This is not intended to be a rigorous test of the @@ -11,7 +11,7 @@ declare zeroext i1 @return_i1() define i1 @test(i32 addrspace(1)* %ptr_base, i32 %arg) gc "statepoint-example" { -; CHECK-LABEL: test +; CHECK-LABEL: test: ; Do we see two spills for the local values and the store to the ; alloca? ; CHECK: subq $40, %rsp @@ -25,11 +25,11 @@ entry: %metadata1 = alloca i32 addrspace(1)*, i32 2, align 8 store i32 addrspace(1)* null, i32 addrspace(1)** %metadata1 %ptr_derived = getelementptr i32, i32 addrspace(1)* %ptr_base, i32 %arg - %safepoint_token = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 2, i32 addrspace(1)* %ptr_base, i32 addrspace(1)* null, i32 addrspace(1)* %ptr_base, i32 addrspace(1)* %ptr_derived, i32 addrspace(1)* null) - %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(i32 %safepoint_token) - %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 9, i32 9) - %b = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 9, i32 10) - %c = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 11, i32 11) + %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 2, i32 addrspace(1)* %ptr_base, i32 addrspace(1)* null, i32 addrspace(1)* %ptr_base, i32 addrspace(1)* %ptr_derived, i32 addrspace(1)* null) + %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token) + %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 9, i32 9) + %b = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 9, i32 10) + %c = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 11, i32 11) ; ret i1 %call1 } @@ -53,11 +53,11 @@ define i1 @test_derived_arg(i32 addrspace(1)* %ptr_base, entry: %metadata1 = alloca i32 addrspace(1)*, i32 2, align 8 store i32 addrspace(1)* null, i32 addrspace(1)** %metadata1 - %safepoint_token = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 2, i32 addrspace(1)* %ptr_base, i32 addrspace(1)* null, i32 addrspace(1)* %ptr_base, i32 addrspace(1)* %ptr_derived, i32 addrspace(1)* null) - %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(i32 %safepoint_token) - %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 9, i32 9) - %b = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 9, i32 10) - %c = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32 %safepoint_token, i32 11, i32 11) + %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 2, i32 addrspace(1)* %ptr_base, i32 addrspace(1)* null, i32 addrspace(1)* %ptr_base, i32 addrspace(1)* %ptr_derived, i32 addrspace(1)* null) + %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token) + %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 9, i32 9) + %b = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 9, i32 10) + %c = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token, i32 11, i32 11) ; ret i1 %call1 } @@ -66,15 +66,15 @@ entry: define i1 @test_id() gc "statepoint-example" { ; CHECK-LABEL: test_id entry: - %safepoint_token = tail call i32 (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 237, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) - %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(i32 %safepoint_token) + %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 237, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0) + %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token) ret i1 %call1 } -declare i32 @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...) -declare i1 @llvm.experimental.gc.result.i1(i32) -declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) #3 +declare token @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...) +declare i1 @llvm.experimental.gc.result.i1(token) +declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32, i32) #3 ; CHECK-LABEL: .section .llvm_stackmaps ; CHECK-NEXT: __LLVM_StackMaps: @@ -94,18 +94,19 @@ declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) #3 ; CHECK-NEXT: .quad 40 ; CHECK-NEXT: .quad test_derived_arg ; CHECK-NEXT: .quad 40 +; CHECK-NEXT: .quad test_id +; CHECK-NEXT: .quad 8 ; ; test ; -; Large Constants -; Statepoint ID only -; CHECK: .quad 0 +; Statepoint ID +; CHECK-NEXT: .quad 0 ; Callsites ; Constant arguments -; CHECK: .long .Ltmp1-test +; CHECK-NEXT: .long .Ltmp1-test ; CHECK: .short 0 ; CHECK: .short 11 ; SmallConstant (0) @@ -123,8 +124,8 @@ declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) #3 ; CHECK: .byte 8 ; CHECK: .short 0 ; CHECK: .long 2 -; Direct Spill Slot [RSP+0] -; CHECK: .byte 2 +; Indirect Spill Slot [RSP+0] +; CHECK: .byte 3 ; CHECK: .byte 8 ; CHECK: .short 7 ; CHECK: .long 16 @@ -143,23 +144,23 @@ declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) #3 ; CHECK: .byte 8 ; CHECK: .short 0 ; CHECK: .long 0 -; Direct Spill Slot [RSP+16] -; CHECK: .byte 2 +; Indirect Spill Slot [RSP+16] +; CHECK: .byte 3 ; CHECK: .byte 8 ; CHECK: .short 7 ; CHECK: .long 16 -; Direct Spill Slot [RSP+8] -; CHECK: .byte 2 +; Indirect Spill Slot [RSP+8] +; CHECK: .byte 3 ; CHECK: .byte 8 ; CHECK: .short 7 ; CHECK: .long 8 -; Direct Spill Slot [RSP+16] -; CHECK: .byte 2 +; Indirect Spill Slot [RSP+16] +; CHECK: .byte 3 ; CHECK: .byte 8 ; CHECK: .short 7 ; CHECK: .long 16 -; Direct Spill Slot [RSP+16] -; CHECK: .byte 2 +; Indirect Spill Slot [RSP+16] +; CHECK: .byte 3 ; CHECK: .byte 8 ; CHECK: .short 7 ; CHECK: .long 16 @@ -171,15 +172,13 @@ declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) #3 ; ; test_derived_arg -; -; Large Constants -; Statepoint ID only -; CHECK: .quad 0 +; Statepoint ID +; CHECK-NEXT: .quad 0 ; Callsites ; Constant arguments -; CHECK: .long .Ltmp3-test_derived_arg +; CHECK-NEXT: .long .Ltmp3-test_derived_arg ; CHECK: .short 0 ; CHECK: .short 11 ; SmallConstant (0) @@ -192,8 +191,8 @@ declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) #3 ; CHECK: .byte 8 ; CHECK: .short 0 ; CHECK: .long 2 -; Direct Spill Slot [RSP+0] -; CHECK: .byte 2 +; Indirect Spill Slot [RSP+0] +; CHECK: .byte 3 ; CHECK: .byte 8 ; CHECK: .short 7 ; CHECK: .long 16 @@ -212,23 +211,23 @@ declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) #3 ; CHECK: .byte 8 ; CHECK: .short 0 ; CHECK: .long 0 -; Direct Spill Slot [RSP+16] -; CHECK: .byte 2 +; Indirect Spill Slot [RSP+16] +; CHECK: .byte 3 ; CHECK: .byte 8 ; CHECK: .short 7 ; CHECK: .long 16 -; Direct Spill Slot [RSP+8] -; CHECK: .byte 2 +; Indirect Spill Slot [RSP+8] +; CHECK: .byte 3 ; CHECK: .byte 8 ; CHECK: .short 7 ; CHECK: .long 8 -; Direct Spill Slot [RSP+16] -; CHECK: .byte 2 +; Indirect Spill Slot [RSP+16] +; CHECK: .byte 3 ; CHECK: .byte 8 ; CHECK: .short 7 ; CHECK: .long 16 -; Direct Spill Slot [RSP+16] -; CHECK: .byte 2 +; Indirect Spill Slot [RSP+16] +; CHECK: .byte 3 ; CHECK: .byte 8 ; CHECK: .short 7 ; CHECK: .long 16 @@ -239,13 +238,12 @@ declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(i32, i32, i32) #3 ; CHECK: .align 8 ; Records for the test_id function: -; No large constants ; The Statepoint ID: -; CHECK: .quad 237 +; CHECK-NEXT: .quad 237 ; Instruction Offset -; CHECK: .long .Ltmp5-test_id +; CHECK-NEXT: .long .Ltmp5-test_id ; Reserved: ; CHECK: .short 0 diff --git a/test/CodeGen/X86/stdarg.ll b/test/CodeGen/X86/stdarg.ll index 18d502ad5834..42cbcb1008d3 100644 --- a/test/CodeGen/X86/stdarg.ll +++ b/test/CodeGen/X86/stdarg.ll @@ -1,5 +1,4 @@ ; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s -; CHECK: testb %al, %al %struct.__va_list_tag = type { i32, i32, i8*, i8* } @@ -8,6 +7,15 @@ entry: %ap = alloca [1 x %struct.__va_list_tag], align 8; <[1 x %struct.__va_list_tag]*> [#uses=2] %ap12 = bitcast [1 x %struct.__va_list_tag]* %ap to i8*; [#uses=2] call void @llvm.va_start(i8* %ap12) +; CHECK: testb %al, %al + +; These test for specific offsets, which is very fragile. Still, the test needs +; to ensure that va_list has the correct element types. +; +; CHECK-DAG: movq {{.*}}, 192(%rsp) +; CHECK-DAG: movq {{.*}}, 184(%rsp) +; CHECK-DAG: movl {{.*}}, 180(%rsp) +; CHECK-DAG: movl {{.*}}, 176(%rsp) %ap3 = getelementptr inbounds [1 x %struct.__va_list_tag], [1 x %struct.__va_list_tag]* %ap, i64 0, i64 0; <%struct.__va_list_tag*> [#uses=1] call void @bar(%struct.__va_list_tag* %ap3) nounwind call void @llvm.va_end(i8* %ap12) diff --git a/test/CodeGen/X86/stores-merging.ll b/test/CodeGen/X86/stores-merging.ll index d6daa573b4ae..9e479bd71b98 100644 --- a/test/CodeGen/X86/stores-merging.ll +++ b/test/CodeGen/X86/stores-merging.ll @@ -7,17 +7,51 @@ target triple = "x86_64-unknown-linux-gnu" @e = common global %structTy zeroinitializer, align 4 -; CHECK-LABEL: f -define void @f() { -entry: +;; Ensure that MergeConsecutiveStores doesn't incorrectly reorder +;; store operations. The first test stores in increasing address +;; order, the second in decreasing -- but in both cases should have +;; the same result in memory in the end. -; CHECK: movabsq $528280977409, %rax +; CHECK-LABEL: redundant_stores_merging: +; CHECK: movl $123, e+8(%rip) +; CHECK: movabsq $1958505086977, %rax ; CHECK: movq %rax, e+4(%rip) -; CHECK: movl $456, e+8(%rip) - +define void @redundant_stores_merging() { +entry: store i32 1, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 1), align 4 store i32 123, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 2), align 4 store i32 456, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 2), align 4 ret void } +;; This variant tests PR25154. +; CHECK-LABEL: redundant_stores_merging_reverse: +; CHECK: movl $123, e+8(%rip) +; CHECK: movabsq $1958505086977, %rax +; CHECK: movq %rax, e+4(%rip) +define void @redundant_stores_merging_reverse() { +entry: + store i32 123, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 2), align 4 + store i32 456, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 2), align 4 + store i32 1, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 1), align 4 + ret void +} + +@b = common global [8 x i8] zeroinitializer, align 2 + +;; The 2-byte store to offset 3 overlaps the 2-byte store to offset 2; +;; these must not be reordered in MergeConsecutiveStores such that the +;; store to 3 comes first (e.g. by merging the stores to 0 and 2 into +;; a movl, after the store to 3). + +;; CHECK-LABEL: overlapping_stores_merging: +;; CHECK: movw $0, b+2(%rip) +;; CHECK: movw $2, b+3(%rip) +;; CHECK: movw $1, b(%rip) +define void @overlapping_stores_merging() { +entry: + store i16 0, i16* bitcast (i8* getelementptr inbounds ([8 x i8], [8 x i8]* @b, i64 0, i64 2) to i16*), align 2 + store i16 2, i16* bitcast (i8* getelementptr inbounds ([8 x i8], [8 x i8]* @b, i64 0, i64 3) to i16*), align 1 + store i16 1, i16* bitcast (i8* getelementptr inbounds ([8 x i8], [8 x i8]* @b, i64 0, i64 0) to i16*), align 2 + ret void +} diff --git a/test/CodeGen/X86/switch-bt.ll b/test/CodeGen/X86/switch-bt.ll index 2cf3aafe5471..6a2cbe1ec6ca 100644 --- a/test/CodeGen/X86/switch-bt.ll +++ b/test/CodeGen/X86/switch-bt.ll @@ -5,7 +5,7 @@ ; CHECK: movabsq $2305843009482129440, %r ; CHECK-NEXT: btq %rax, %r -; CHECK-NEXT: jae +; CHECK-NEXT: jb ; CHECK: movl $671088640, %e ; CHECK-NEXT: btq %rax, %r ; CHECK-NEXT: jae @@ -145,13 +145,13 @@ sw.epilog: ; CHECK: cmpl $10 ; CHECK: je ; CHECK: cmpl $20 +; CHECK: je +; CHECK: cmpl $30 ; CHECK: jne ; CHECK: cmpl $40 ; CHECK: je ; CHECK: cmpl $50 -; CHECK: jne -; CHECK: cmpl $30 -; CHECK: jne +; CHECK: je ; CHECK: cmpl $60 ; CHECK: jne } diff --git a/test/CodeGen/X86/switch-edge-weight.ll b/test/CodeGen/X86/switch-edge-weight.ll new file mode 100644 index 000000000000..b8cb7b1280ad --- /dev/null +++ b/test/CodeGen/X86/switch-edge-weight.ll @@ -0,0 +1,281 @@ +; RUN: llc -march=x86-64 -print-machineinstrs=expand-isel-pseudos %s -o /dev/null 2>&1 | FileCheck %s + +declare void @foo(i32) + +; CHECK-LABEL: test + +define void @test(i32 %x) nounwind { +entry: + switch i32 %x, label %sw.default [ + i32 1, label %sw.bb + i32 155, label %sw.bb + i32 156, label %sw.bb + i32 157, label %sw.bb + i32 158, label %sw.bb + i32 159, label %sw.bb + i32 1134, label %sw.bb + i32 1140, label %sw.bb + ], !prof !1 + +sw.bb: + call void @foo(i32 0) + br label %sw.epilog + +sw.default: + call void @foo(i32 1) + br label %sw.epilog + +sw.epilog: + ret void + +; Check if weights are correctly assigned to edges generated from switch +; statement. +; +; CHECK: BB#0: +; BB#0 to BB#4: [0, 1133] (65 = 60 + 5) +; BB#0 to BB#5: [1134, UINT32_MAX] (25 = 20 + 5) +; CHECK: Successors according to CFG: BB#4({{[0-9a-fx/= ]+}}72.22%) BB#5({{[0-9a-fx/= ]+}}27.78%) +; +; CHECK: BB#4: +; BB#4 to BB#1: [155, 159] (50) +; BB#4 to BB#5: [0, 1133] - [155, 159] (15 = 10 + 5) +; CHECK: Successors according to CFG: BB#1({{[0-9a-fx/= ]+}}76.92%) BB#7({{[0-9a-fx/= ]+}}23.08%) +; +; CHECK: BB#5: +; BB#5 to BB#1: {1140} (10) +; BB#5 to BB#6: [1134, UINT32_MAX] - {1140} (15 = 10 + 5) +; CHECK: Successors according to CFG: BB#1({{[0-9a-fx/= ]+}}40.00%) BB#6({{[0-9a-fx/= ]+}}60.00%) +; +; CHECK: BB#6: +; BB#6 to BB#1: {1134} (10) +; BB#6 to BB#2: [1134, UINT32_MAX] - {1134, 1140} (5) +; CHECK: Successors according to CFG: BB#1({{[0-9a-fx/= ]+}}66.67%) BB#2({{[0-9a-fx/= ]+}}33.33%) +} + +; CHECK-LABEL: test2 + +define void @test2(i32 %x) nounwind { +entry: + +; In this switch statement, there is an edge from jump table to default +; statement. + + switch i32 %x, label %sw.default [ + i32 1, label %sw.bb + i32 10, label %sw.bb2 + i32 11, label %sw.bb3 + i32 12, label %sw.bb4 + i32 13, label %sw.bb5 + i32 14, label %sw.bb5 + ], !prof !3 + +sw.bb: + call void @foo(i32 0) + br label %sw.epilog + +sw.bb2: + call void @foo(i32 2) + br label %sw.epilog + +sw.bb3: + call void @foo(i32 3) + br label %sw.epilog + +sw.bb4: + call void @foo(i32 4) + br label %sw.epilog + +sw.bb5: + call void @foo(i32 5) + br label %sw.epilog + +sw.default: + call void @foo(i32 1) + br label %sw.epilog + +sw.epilog: + ret void + +; Check if weights are correctly assigned to edges generated from switch +; statement. +; +; CHECK: BB#0: +; BB#0 to BB#6: {0} + [15, UINT32_MAX] (5) +; BB#0 to BB#8: [1, 14] (jump table) (65 = 60 + 5) +; CHECK: Successors according to CFG: BB#6({{[0-9a-fx/= ]+}}7.14%) BB#8({{[0-9a-fx/= ]+}}92.86% +; +; CHECK: BB#8: +; BB#8 to BB#1: {1} (10) +; BB#8 to BB#6: [2, 9] (5) +; BB#8 to BB#2: {10} (10) +; BB#8 to BB#3: {11} (10) +; BB#8 to BB#4: {12} (10) +; BB#8 to BB#5: {13, 14} (20) +; CHECK: Successors according to CFG: BB#1({{[0-9a-fx/= ]+}}15.38%) BB#6({{[0-9a-fx/= ]+}}7.69%) BB#2({{[0-9a-fx/= ]+}}15.38%) BB#3({{[0-9a-fx/= ]+}}15.38%) BB#4({{[0-9a-fx/= ]+}}15.38%) BB#5({{[0-9a-fx/= ]+}}30.77%) +} + +; CHECK-LABEL: test3 + +define void @test3(i32 %x) nounwind { +entry: + +; In this switch statement, there is no edge from jump table to default +; statement. + + switch i32 %x, label %sw.default [ + i32 10, label %sw.bb + i32 11, label %sw.bb2 + i32 12, label %sw.bb3 + i32 13, label %sw.bb4 + i32 14, label %sw.bb5 + ], !prof !2 + +sw.bb: + call void @foo(i32 0) + br label %sw.epilog + +sw.bb2: + call void @foo(i32 2) + br label %sw.epilog + +sw.bb3: + call void @foo(i32 3) + br label %sw.epilog + +sw.bb4: + call void @foo(i32 4) + br label %sw.epilog + +sw.bb5: + call void @foo(i32 5) + br label %sw.epilog + +sw.default: + call void @foo(i32 1) + br label %sw.epilog + +sw.epilog: + ret void + +; Check if weights are correctly assigned to edges generated from switch +; statement. +; +; CHECK: BB#0: +; BB#0 to BB#6: [0, 9] + [15, UINT32_MAX] {10} +; BB#0 to BB#8: [10, 14] (jump table) (50) +; CHECK: Successors according to CFG: BB#6({{[0-9a-fx/= ]+}}16.67%) BB#8({{[0-9a-fx/= ]+}}83.33%) +; +; CHECK: BB#8: +; BB#8 to BB#1: {10} (10) +; BB#8 to BB#2: {11} (10) +; BB#8 to BB#3: {12} (10) +; BB#8 to BB#4: {13} (10) +; BB#8 to BB#5: {14} (10) +; CHECK: Successors according to CFG: BB#1({{[0-9a-fx/= ]+}}20.00%) BB#2({{[0-9a-fx/= ]+}}20.00%) BB#3({{[0-9a-fx/= ]+}}20.00%) BB#4({{[0-9a-fx/= ]+}}20.00%) BB#5({{[0-9a-fx/= ]+}}20.00%) +} + +; CHECK-LABEL: test4 + +define void @test4(i32 %x) nounwind { +entry: + +; In this switch statement, there is no edge from bit test to default basic +; block. + + switch i32 %x, label %sw.default [ + i32 1, label %sw.bb + i32 111, label %sw.bb2 + i32 112, label %sw.bb3 + i32 113, label %sw.bb3 + i32 114, label %sw.bb2 + i32 115, label %sw.bb2 + ], !prof !3 + +sw.bb: + call void @foo(i32 0) + br label %sw.epilog + +sw.bb2: + call void @foo(i32 2) + br label %sw.epilog + +sw.bb3: + call void @foo(i32 3) + br label %sw.epilog + +sw.default: + call void @foo(i32 1) + br label %sw.epilog + +sw.epilog: + ret void + +; Check if weights are correctly assigned to edges generated from switch +; statement. +; +; CHECK: BB#0: +; BB#0 to BB#6: [0, 110] + [116, UINT32_MAX] (20) +; BB#0 to BB#7: [111, 115] (bit test) (50) +; CHECK: Successors according to CFG: BB#6({{[0-9a-fx/= ]+}}28.57%) BB#7({{[0-9a-fx/= ]+}}71.43%) +; +; CHECK: BB#7: +; BB#7 to BB#2: {111, 114, 115} (30) +; BB#7 to BB#3: {112, 113} (20) +; CHECK: Successors according to CFG: BB#2({{[0-9a-fx/= ]+}}60.00%) BB#3({{[0-9a-fx/= ]+}}40.00%) +} + +; CHECK-LABEL: test5 + +define void @test5(i32 %x) nounwind { +entry: + +; In this switch statement, there is an edge from jump table to default basic +; block. + + switch i32 %x, label %sw.default [ + i32 1, label %sw.bb + i32 5, label %sw.bb2 + i32 7, label %sw.bb3 + i32 9, label %sw.bb4 + i32 31, label %sw.bb5 + ], !prof !2 + +sw.bb: + call void @foo(i32 0) + br label %sw.epilog + +sw.bb2: + call void @foo(i32 1) + br label %sw.epilog + +sw.bb3: + call void @foo(i32 2) + br label %sw.epilog + +sw.bb4: + call void @foo(i32 3) + br label %sw.epilog + +sw.bb5: + call void @foo(i32 4) + br label %sw.epilog + +sw.default: + call void @foo(i32 5) + br label %sw.epilog + +sw.epilog: + ret void + +; Check if weights are correctly assigned to edges generated from switch +; statement. +; +; CHECK: BB#0: +; BB#0 to BB#6: [10, UINT32_MAX] (15) +; BB#0 to BB#8: [1, 5, 7, 9] (jump table) (45) +; CHECK: Successors according to CFG: BB#8({{[0-9a-fx/= ]+}}25.00%) BB#9({{[0-9a-fx/= ]+}}75.00%) +} + +!1 = !{!"branch_weights", i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10} +!2 = !{!"branch_weights", i32 10, i32 10, i32 10, i32 10, i32 10, i32 10} +!3 = !{!"branch_weights", i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10} diff --git a/test/CodeGen/X86/switch-jump-table.ll b/test/CodeGen/X86/switch-jump-table.ll index a84fb4aafd17..896a067da230 100644 --- a/test/CodeGen/X86/switch-jump-table.ll +++ b/test/CodeGen/X86/switch-jump-table.ll @@ -1,17 +1,18 @@ -; RUN: llc -mtriple=i686-pc-gnu-linux < %s | FileCheck %s +; RUN: llc -mtriple=i686-pc-gnu-linux < %s | FileCheck %s -check-prefix=CHECK +; RUN: llc -mtriple=i686-pc-gnu-linux -print-machineinstrs=expand-isel-pseudos %s -o /dev/null 2>&1 | FileCheck %s -check-prefix=CHECK-JT-PROB ; An unreachable default destination is replaced with the most popular case label. -define void @sum2(i32 %x, i32* %to) { -; CHECK-LABEL: sum2: +define void @foo(i32 %x, i32* %to) { +; CHECK-LABEL: foo: ; CHECK: movl 4(%esp), [[REG:%e[a-z]{2}]] ; CHECK: cmpl $3, [[REG]] -; CHECK: jbe .LBB0_1 +; CHECK: ja .LBB0_6 +; CHECK-NEXT: # BB#1: +; CHECK-NEXT: jmpl *.LJTI0_0(,[[REG]],4) ; CHECK: movl $4 ; CHECK: retl -; CHECK-LABEL: .LBB0_1: -; CHECK-NEXT: jmpl *.LJTI0_0(,[[REG]],4) entry: switch i32 %x, label %default [ @@ -48,5 +49,44 @@ default: ; CHECK-NEXT: .long .LBB0_3 ; CHECK-NEXT: .long .LBB0_4 ; CHECK-NEXT: .long .LBB0_5 -; CHECK-NOT: .long } + +; Check if branch probabilities are correctly assigned to the jump table. + +define void @bar(i32 %x, i32* %to) { +; CHECK-JT-PROB-LABEL: bar: +; CHECK-JT-PROB: Successors according to CFG: BB#6({{[0-9a-fx/= ]+}}14.29%) BB#8({{[0-9a-fx/= ]+}}85.71%) +; CHECK-JT-PROB: Successors according to CFG: BB#1({{[0-9a-fx/= ]+}}16.67%) BB#2({{[0-9a-fx/= ]+}}16.67%) BB#3({{[0-9a-fx/= ]+}}16.67%) BB#4({{[0-9a-fx/= ]+}}16.67%) BB#5({{[0-9a-fx/= ]+}}33.33%) + +entry: + switch i32 %x, label %default [ + i32 0, label %bb0 + i32 1, label %bb1 + i32 2, label %bb2 + i32 3, label %bb3 + i32 4, label %bb4 + i32 5, label %bb4 + ], !prof !1 +bb0: + store i32 0, i32* %to + br label %exit +bb1: + store i32 1, i32* %to + br label %exit +bb2: + store i32 2, i32* %to + br label %exit +bb3: + store i32 3, i32* %to + br label %exit +bb4: + store i32 4, i32* %to + br label %exit +default: + store i32 5, i32* %to + br label %exit +exit: + ret void +} + +!1 = !{!"branch_weights", i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16} diff --git a/test/CodeGen/X86/switch-order-weight.ll b/test/CodeGen/X86/switch-order-weight.ll index 207e0b3f707b..8c0c1a7d8108 100644 --- a/test/CodeGen/X86/switch-order-weight.ll +++ b/test/CodeGen/X86/switch-order-weight.ll @@ -13,8 +13,8 @@ entry: ; CHECK-LABEL: test1: ; CHECK-NOT: unr ; CHECK: cmpl $10 -; CHECK: bar ; CHECK: cmpl $20 +; CHECK: bar if.then: tail call void @unr(i32 23) noreturn nounwind diff --git a/test/CodeGen/X86/switch.ll b/test/CodeGen/X86/switch.ll index 748fd6f238b1..46587341ea74 100644 --- a/test/CodeGen/X86/switch.ll +++ b/test/CodeGen/X86/switch.ll @@ -51,7 +51,7 @@ return: ret void ; CHECK-LABEL: simple_ranges ; CHECK: leal -100 ; CHECK: cmpl $4 -; CHECK: jae +; CHECK: jb ; CHECK: cmpl $3 ; CHECK: ja @@ -90,7 +90,7 @@ return: ret void ; but with 6-8, the whole switch is suitable for a jump table. ; CHECK-LABEL: jt_is_better ; CHECK: cmpl $8 -; CHECK: jbe +; CHECK: ja ; CHECK: jmpq *.LJTI } @@ -107,7 +107,6 @@ entry: i32 2, label %bb2 i32 5, label %bb2 i32 8, label %bb2 - ] bb0: tail call void @g(i32 0) br label %return bb1: tail call void @g(i32 1) br label %return @@ -116,6 +115,10 @@ return: ret void ; This could be lowered as a jump table, but bit tests is more efficient. ; CHECK-LABEL: bt_is_better +; The bit test on 2,5,8 is unnecessary as all cases cover the rage [0, 8]. +; The range check guarantees that cases other than 0,3,6 and 1,4,7 must be +; in 2,5,8. +; ; 73 = 2^0 + 2^3 + 2^6 ; CHECK: movl $73 ; CHECK: btl @@ -123,7 +126,74 @@ return: ret void ; CHECK: movl $146 ; CHECK: btl ; 292 = 2^2 + 2^5 + 2^8 -; CHECK: movl $292 +; CHECK-NOT: movl $292 +; CHECK-NOT: btl +} + +define void @bt_is_better2(i32 %x) { +entry: + switch i32 %x, label %return [ + i32 0, label %bb0 + i32 3, label %bb0 + i32 6, label %bb0 + i32 1, label %bb1 + i32 4, label %bb1 + i32 7, label %bb1 + i32 2, label %bb2 + i32 8, label %bb2 + ] +bb0: tail call void @g(i32 0) br label %return +bb1: tail call void @g(i32 1) br label %return +bb2: tail call void @g(i32 2) br label %return +return: ret void + +; This will also be lowered as bit test, but as the range [0,8] is not fully +; covered (5 missing), the default statement can be jumped to and we end up +; with one more branch. +; CHECK-LABEL: bt_is_better2 +; +; 73 = 2^0 + 2^3 + 2^6 +; CHECK: movl $73 +; CHECK: btl +; 146 = 2^1 + 2^4 + 2^7 +; CHECK: movl $146 +; CHECK: btl +; 260 = 2^2 + 2^8 +; CHECK: movl $260 +; CHECK: btl +} + +define void @bt_is_better3(i32 %x) { +entry: + switch i32 %x, label %return [ + i32 10, label %bb0 + i32 13, label %bb0 + i32 16, label %bb0 + i32 11, label %bb1 + i32 14, label %bb1 + i32 17, label %bb1 + i32 12, label %bb2 + i32 18, label %bb2 + ] +bb0: tail call void @g(i32 0) br label %return +bb1: tail call void @g(i32 1) br label %return +bb2: tail call void @g(i32 2) br label %return +return: ret void + +; We don't have to subtract 10 from the case value to let the range become +; [0, 8], as each value in the range [10, 18] can be represented by bits in a +; word. Then we still need a branch to jump to the default statement for the +; range [0, 10). +; CHECK-LABEL: bt_is_better3 +; +; 74752 = 2^10 + 2^13 + 2^16 +; CHECK: movl $74752 +; CHECK: btl +; 149504 = 2^11 + 2^14 + 2^17 +; CHECK: movl $149504 +; CHECK: btl +; 266240 = 2^12 + 2^15 + 2^18 +; CHECK: movl $266240 ; CHECK: btl } @@ -410,6 +480,9 @@ return: ret void ; Cases 1,4,7 have a very large branch weight (which shouldn't overflow), so ; their bit test should come first. 0,3,6 and 2,5,8,9 both have a weight of 12, ; but the latter set has more cases, so should be tested for earlier. +; The bit test on 0,3,6 is unnecessary as all cases cover the rage [0, 9]. +; The range check guarantees that cases other than 1,4,7 and 2,5,8,9 must be +; in 0,3,6. ; CHECK-LABEL: bt_order_by_weight ; 146 = 2^1 + 2^4 + 2^7 @@ -419,8 +492,8 @@ return: ret void ; CHECK: movl $804 ; CHECK: btl ; 73 = 2^0 + 2^3 + 2^6 -; CHECK: movl $73 -; CHECK: btl +; CHECK-NOT: movl $73 +; CHECK-NOT: btl } !1 = !{!"branch_weights", diff --git a/test/CodeGen/X86/swizzle-2.ll b/test/CodeGen/X86/swizzle-2.ll index 697af843abb1..fd81573edec9 100644 --- a/test/CodeGen/X86/swizzle-2.ll +++ b/test/CodeGen/X86/swizzle-2.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 | FileCheck %s ; Test that we correctly fold a shuffle that performs a swizzle of another diff --git a/test/CodeGen/X86/system-intrinsics-64-xsave.ll b/test/CodeGen/X86/system-intrinsics-64-xsave.ll new file mode 100644 index 000000000000..feec9516220b --- /dev/null +++ b/test/CodeGen/X86/system-intrinsics-64-xsave.ll @@ -0,0 +1,41 @@ +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xsave | FileCheck %s + +define void @test_xsave(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xsave +; CHECK: movl %edx, %eax +; CHECK: movl %esi, %edx +; CHECK: xsave (%rdi) + call void @llvm.x86.xsave(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xsave(i8*, i32, i32) + +define void @test_xsave64(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xsave64 +; CHECK: movl %edx, %eax +; CHECK: movl %esi, %edx +; CHECK: xsave64 (%rdi) + call void @llvm.x86.xsave64(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xsave64(i8*, i32, i32) + +define void @test_xrstor(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xrstor +; CHECK: movl %edx, %eax +; CHECK: movl %esi, %edx +; CHECK: xrstor (%rdi) + call void @llvm.x86.xrstor(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xrstor(i8*, i32, i32) + +define void @test_xrstor64(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xrstor64 +; CHECK: movl %edx, %eax +; CHECK: movl %esi, %edx +; CHECK: xrstor64 (%rdi) + call void @llvm.x86.xrstor64(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xrstor64(i8*, i32, i32) diff --git a/test/CodeGen/X86/system-intrinsics-64-xsavec.ll b/test/CodeGen/X86/system-intrinsics-64-xsavec.ll new file mode 100644 index 000000000000..068034886515 --- /dev/null +++ b/test/CodeGen/X86/system-intrinsics-64-xsavec.ll @@ -0,0 +1,21 @@ +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xsave,+xsavec | FileCheck %s + +define void @test_xsavec(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xsavec +; CHECK: movl %edx, %eax +; CHECK: movl %esi, %edx +; CHECK: xsavec (%rdi) + call void @llvm.x86.xsavec(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xsavec(i8*, i32, i32) + +define void @test_xsavec64(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xsavec64 +; CHECK: movl %edx, %eax +; CHECK: movl %esi, %edx +; CHECK: xsavec64 (%rdi) + call void @llvm.x86.xsavec64(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xsavec64(i8*, i32, i32) diff --git a/test/CodeGen/X86/system-intrinsics-64-xsaveopt.ll b/test/CodeGen/X86/system-intrinsics-64-xsaveopt.ll new file mode 100644 index 000000000000..ee0a5360da8e --- /dev/null +++ b/test/CodeGen/X86/system-intrinsics-64-xsaveopt.ll @@ -0,0 +1,21 @@ +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xsaveopt | FileCheck %s + +define void @test_xsaveopt(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xsaveopt +; CHECK: movl %edx, %eax +; CHECK: movl %esi, %edx +; CHECK: xsaveopt (%rdi) + call void @llvm.x86.xsaveopt(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xsaveopt(i8*, i32, i32) + +define void @test_xsaveopt64(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xsaveopt64 +; CHECK: movl %edx, %eax +; CHECK: movl %esi, %edx +; CHECK: xsaveopt64 (%rdi) + call void @llvm.x86.xsaveopt64(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xsaveopt64(i8*, i32, i32) diff --git a/test/CodeGen/X86/system-intrinsics-64-xsaves.ll b/test/CodeGen/X86/system-intrinsics-64-xsaves.ll new file mode 100644 index 000000000000..5c1c5be4e7e2 --- /dev/null +++ b/test/CodeGen/X86/system-intrinsics-64-xsaves.ll @@ -0,0 +1,41 @@ +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xsave,+xsaves | FileCheck %s + +define void @test_xsaves(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xsaves +; CHECK: movl %edx, %eax +; CHECK: movl %esi, %edx +; CHECK: xsaves (%rdi) + call void @llvm.x86.xsaves(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xsaves(i8*, i32, i32) + +define void @test_xsaves64(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xsaves64 +; CHECK: movl %edx, %eax +; CHECK: movl %esi, %edx +; CHECK: xsaves64 (%rdi) + call void @llvm.x86.xsaves64(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xsaves64(i8*, i32, i32) + +define void @test_xrstors(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xrstors +; CHECK: movl %edx, %eax +; CHECK: movl %esi, %edx +; CHECK: xrstors (%rdi) + call void @llvm.x86.xrstors(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xrstors(i8*, i32, i32) + +define void @test_xrstors64(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xrstors64 +; CHECK: movl %edx, %eax +; CHECK: movl %esi, %edx +; CHECK: xrstors64 (%rdi) + call void @llvm.x86.xrstors64(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xrstors64(i8*, i32, i32) diff --git a/test/CodeGen/X86/system-intrinsics-64.ll b/test/CodeGen/X86/system-intrinsics-64.ll index 96c441773390..e18a79c2b614 100644 --- a/test/CodeGen/X86/system-intrinsics-64.ll +++ b/test/CodeGen/X86/system-intrinsics-64.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+fxsr | FileCheck %s define void @test_fxsave(i8* %ptr) { ; CHECK-LABEL: test_fxsave diff --git a/test/CodeGen/X86/system-intrinsics-xsave.ll b/test/CodeGen/X86/system-intrinsics-xsave.ll new file mode 100644 index 000000000000..ff9fb7e247a4 --- /dev/null +++ b/test/CodeGen/X86/system-intrinsics-xsave.ll @@ -0,0 +1,23 @@ +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+xsave | FileCheck %s + +define void @test_xsave(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xsave +; CHECK: movl 8(%esp), %edx +; CHECK: movl 12(%esp), %eax +; CHECK: movl 4(%esp), %ecx +; CHECK: xsave (%ecx) + call void @llvm.x86.xsave(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xsave(i8*, i32, i32) + +define void @test_xrstor(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xrstor +; CHECK: movl 8(%esp), %edx +; CHECK: movl 12(%esp), %eax +; CHECK: movl 4(%esp), %ecx +; CHECK: xrstor (%ecx) + call void @llvm.x86.xrstor(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xrstor(i8*, i32, i32) diff --git a/test/CodeGen/X86/system-intrinsics-xsavec.ll b/test/CodeGen/X86/system-intrinsics-xsavec.ll new file mode 100644 index 000000000000..4a55ea9531b1 --- /dev/null +++ b/test/CodeGen/X86/system-intrinsics-xsavec.ll @@ -0,0 +1,12 @@ +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+xsave,+xsavec | FileCheck %s + +define void @test_xsavec(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xsavec +; CHECK: movl 8(%esp), %edx +; CHECK: movl 12(%esp), %eax +; CHECK: movl 4(%esp), %ecx +; CHECK: xsavec (%ecx) + call void @llvm.x86.xsavec(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xsavec(i8*, i32, i32) diff --git a/test/CodeGen/X86/system-intrinsics-xsaveopt.ll b/test/CodeGen/X86/system-intrinsics-xsaveopt.ll new file mode 100644 index 000000000000..f9bd7acd5a7c --- /dev/null +++ b/test/CodeGen/X86/system-intrinsics-xsaveopt.ll @@ -0,0 +1,12 @@ +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+xsave,+xsaveopt | FileCheck %s + +define void @test_xsaveopt(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xsaveopt +; CHECK: movl 8(%esp), %edx +; CHECK: movl 12(%esp), %eax +; CHECK: movl 4(%esp), %ecx +; CHECK: xsaveopt (%ecx) + call void @llvm.x86.xsaveopt(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xsaveopt(i8*, i32, i32) diff --git a/test/CodeGen/X86/system-intrinsics-xsaves.ll b/test/CodeGen/X86/system-intrinsics-xsaves.ll new file mode 100644 index 000000000000..ca1c5c1a9ed0 --- /dev/null +++ b/test/CodeGen/X86/system-intrinsics-xsaves.ll @@ -0,0 +1,23 @@ +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+xsave,+xsaves | FileCheck %s + +define void @test_xsaves(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xsaves +; CHECK: movl 8(%esp), %edx +; CHECK: movl 12(%esp), %eax +; CHECK: movl 4(%esp), %ecx +; CHECK: xsaves (%ecx) + call void @llvm.x86.xsaves(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xsaves(i8*, i32, i32) + +define void @test_xrstors(i8* %ptr, i32 %hi, i32 %lo) { +; CHECK-LABEL: test_xrstors +; CHECK: movl 8(%esp), %edx +; CHECK: movl 12(%esp), %eax +; CHECK: movl 4(%esp), %ecx +; CHECK: xrstors (%ecx) + call void @llvm.x86.xrstors(i8* %ptr, i32 %hi, i32 %lo) + ret void; +} +declare void @llvm.x86.xrstors(i8*, i32, i32) diff --git a/test/CodeGen/X86/system-intrinsics.ll b/test/CodeGen/X86/system-intrinsics.ll index 84fcd052d7db..90dc9cd21e67 100644 --- a/test/CodeGen/X86/system-intrinsics.ll +++ b/test/CodeGen/X86/system-intrinsics.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+fxsr | FileCheck %s define void @test_fxsave(i8* %ptr) { ; CHECK-LABEL: test_fxsave diff --git a/test/CodeGen/X86/tail-dup-catchret.ll b/test/CodeGen/X86/tail-dup-catchret.ll new file mode 100644 index 000000000000..3eeb24d20f2d --- /dev/null +++ b/test/CodeGen/X86/tail-dup-catchret.ll @@ -0,0 +1,31 @@ +; RUN: llc < %s | FileCheck %s +target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32" +target triple = "i686-pc-windows-msvc18.0.0" + +define void @f() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) { +entry: + invoke void @g() + to label %try.cont unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %cs1 = catchswitch within none [label %catch] unwind to caller + +catch: ; preds = %catch.dispatch + %0 = catchpad within %cs1 [i8* null, i32 64, i8* null] + catchret from %0 to label %try.cont + +try.cont: ; preds = %entry, %catch + %b.0 = phi i1 [ false, %catch ], [ true, %entry ] + tail call void @h(i1 zeroext %b.0) + ret void +} + +; CHECK-LABEL: _f: +; CHECK: calll _g +; CHECK: calll _h + +declare void @g() + +declare i32 @__CxxFrameHandler3(...) + +declare void @h(i1 zeroext) diff --git a/test/CodeGen/X86/tail-merge-wineh.ll b/test/CodeGen/X86/tail-merge-wineh.ll new file mode 100644 index 000000000000..69c2fda6949b --- /dev/null +++ b/test/CodeGen/X86/tail-merge-wineh.ll @@ -0,0 +1,107 @@ +; RUN: llc < %s | FileCheck %s + +; Started from this code: +; void f() { +; try { +; try { +; throw 42; +; } catch (int) { +; } +; try { +; throw 42; +; } catch (int) { +; } +; } catch (int) { +; } +; } + +; Don't tail merge the calls. +; CHECK: calll __CxxThrowException@8 +; CHECK: calll __CxxThrowException@8 + +; ModuleID = 'cppeh-pingpong.cpp' +target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32" +target triple = "i686-pc-windows-msvc" + +%rtti.TypeDescriptor2 = type { i8**, i8*, [3 x i8] } +%eh.CatchableType = type { i32, i8*, i32, i32, i32, i32, i8* } +%eh.CatchableTypeArray.1 = type { i32, [1 x %eh.CatchableType*] } +%eh.ThrowInfo = type { i32, i8*, i8*, i8* } + +$"\01??_R0H@8" = comdat any + +$"_CT??_R0H@84" = comdat any + +$_CTA1H = comdat any + +$_TI1H = comdat any + +@"\01??_7type_info@@6B@" = external constant i8* +@"\01??_R0H@8" = linkonce_odr global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".H\00" }, comdat +@"_CT??_R0H@84" = linkonce_odr unnamed_addr constant %eh.CatchableType { i32 1, i8* bitcast (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i8*), i32 0, i32 -1, i32 0, i32 4, i8* null }, section ".xdata", comdat +@_CTA1H = linkonce_odr unnamed_addr constant %eh.CatchableTypeArray.1 { i32 1, [1 x %eh.CatchableType*] [%eh.CatchableType* @"_CT??_R0H@84"] }, section ".xdata", comdat +@_TI1H = linkonce_odr unnamed_addr constant %eh.ThrowInfo { i32 0, i8* null, i8* null, i8* bitcast (%eh.CatchableTypeArray.1* @_CTA1H to i8*) }, section ".xdata", comdat + +define void @"\01?f@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) { +entry: + %i = alloca i32, align 4 + %tmp = alloca i32, align 4 + %tmp1 = alloca i32, align 4 + store i32 0, i32* %i, align 4 + store i32 42, i32* %tmp, align 4 + %0 = bitcast i32* %tmp to i8* + invoke void @_CxxThrowException(i8* %0, %eh.ThrowInfo* @_TI1H) #1 + to label %unreachable unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %cs1 = catchswitch within none [label %catch] unwind label %catch.dispatch.7 + +catch: ; preds = %catch.dispatch + %1 = catchpad within %cs1 [%rtti.TypeDescriptor2* @"\01??_R0H@8", i32 0, i8* null] + catchret from %1 to label %catchret.dest + +catchret.dest: ; preds = %catch + br label %try.cont + +try.cont: ; preds = %catchret.dest + store i32 42, i32* %tmp1, align 4 + %2 = bitcast i32* %tmp1 to i8* + invoke void @_CxxThrowException(i8* %2, %eh.ThrowInfo* @_TI1H) #1 + to label %unreachable unwind label %catch.dispatch.2 + +catch.dispatch.2: ; preds = %try.cont + %cs2 = catchswitch within none [label %catch.4] unwind label %catch.dispatch.7 + +catch.4: ; preds = %catch.dispatch.2 + %3 = catchpad within %cs2 [%rtti.TypeDescriptor2* @"\01??_R0H@8", i32 0, i8* null] + catchret from %3 to label %catchret.dest.5 + +catchret.dest.5: ; preds = %catch.4 + br label %try.cont.6 + +try.cont.6: ; preds = %catchret.dest.5 + br label %try.cont.11 + +catch.dispatch.7: + %cs3 = catchswitch within none [label %catch.9] unwind to caller + +catch.9: ; preds = %catch.dispatch.7 + %4 = catchpad within %cs3 [%rtti.TypeDescriptor2* @"\01??_R0H@8", i32 0, i8* null] + catchret from %4 to label %catchret.dest.10 + +catchret.dest.10: ; preds = %catch.9 + br label %try.cont.11 + +try.cont.11: ; preds = %catchret.dest.10, %try.cont.6 + ret void + +unreachable: ; preds = %try.cont, %entry + unreachable +} + +declare x86_stdcallcc void @_CxxThrowException(i8*, %eh.ThrowInfo*) + +declare i32 @__CxxFrameHandler3(...) + +attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { noreturn } diff --git a/test/CodeGen/X86/tail-opts.ll b/test/CodeGen/X86/tail-opts.ll index f590176d9815..bf778e5bad2b 100644 --- a/test/CodeGen/X86/tail-opts.ll +++ b/test/CodeGen/X86/tail-opts.ll @@ -277,8 +277,8 @@ declare fastcc %union.tree_node* @default_conversion(%union.tree_node*) nounwind ; CHECK-LABEL: foo: ; CHECK: callq func -; CHECK-NEXT: .LBB4_2: ; CHECK-NEXT: popq +; CHECK-NEXT: .LBB4_2: ; CHECK-NEXT: ret define void @foo(i1* %V) nounwind { @@ -371,6 +371,44 @@ return: ret void } +; two_minsize - Same as two, but with minsize instead of optsize. + +; CHECK-LABEL: two_minsize: +; CHECK-NOT: XYZ +; CHECK: ret +; CHECK: movl $0, XYZ(%rip) +; CHECK: movl $1, XYZ(%rip) +; CHECK-NOT: XYZ + +define void @two_minsize() nounwind minsize { +entry: + %0 = icmp eq i32 undef, 0 + br i1 %0, label %bbx, label %bby + +bby: + switch i32 undef, label %bb7 [ + i32 16, label %return + ] + +bb7: + store volatile i32 0, i32* @XYZ + store volatile i32 1, i32* @XYZ + unreachable + +bbx: + switch i32 undef, label %bb12 [ + i32 128, label %return + ] + +bb12: + store volatile i32 0, i32* @XYZ + store volatile i32 1, i32* @XYZ + unreachable + +return: + ret void +} + ; two_nosize - Same as two, but without the optsize attribute. ; Now two instructions are enough to be tail-duplicated. diff --git a/test/CodeGen/X86/tailcall-mem-intrinsics.ll b/test/CodeGen/X86/tailcall-mem-intrinsics.ll index 0e0ab5c478fc..8e1e4f464baa 100644 --- a/test/CodeGen/X86/tailcall-mem-intrinsics.ll +++ b/test/CodeGen/X86/tailcall-mem-intrinsics.ll @@ -8,8 +8,8 @@ entry: ret void } -; CHECK-LABEL: tail_memset -; CHECK; jmp memmove +; CHECK-LABEL: tail_memmove +; CHECK: jmp memmove define void @tail_memmove(i8* nocapture %p, i8* nocapture readonly %q, i32 %n) #0 { entry: tail call void @llvm.memmove.p0i8.p0i8.i32(i8* %p, i8* %q, i32 %n, i32 1, i1 false) diff --git a/test/CodeGen/X86/tailcall-msvc-conventions.ll b/test/CodeGen/X86/tailcall-msvc-conventions.ll new file mode 100644 index 000000000000..98b02c9c07e8 --- /dev/null +++ b/test/CodeGen/X86/tailcall-msvc-conventions.ll @@ -0,0 +1,189 @@ +; RUN: llc -mtriple=i686-unknown-linux-gnu -O1 < %s | FileCheck %s +; RUN: llc -mtriple=i686-unknown-linux-gnu -O0 < %s | FileCheck %s + +; The MSVC family of x86 calling conventions makes tail calls really tricky. +; Tests of all the various combinations should live here. + +declare i32 @cdecl_i32() +declare void @cdecl_void() + +; Don't allow tail calling these cdecl functions, because we need to clear the +; incoming stack arguments for these argument-clearing conventions. + +define x86_thiscallcc void @thiscall_cdecl_notail(i32 %a, i32 %b, i32 %c) { + tail call void @cdecl_void() + ret void +} +; CHECK-LABEL: thiscall_cdecl_notail +; CHECK: calll cdecl_void +; CHECK: retl $8 + +define x86_stdcallcc void @stdcall_cdecl_notail(i32 %a, i32 %b, i32 %c) { + tail call void @cdecl_void() + ret void +} +; CHECK-LABEL: stdcall_cdecl_notail +; CHECK: calll cdecl_void +; CHECK: retl $12 + +define x86_vectorcallcc void @vectorcall_cdecl_notail(i32 inreg %a, i32 inreg %b, i32 %c) { + tail call void @cdecl_void() + ret void +} +; CHECK-LABEL: vectorcall_cdecl_notail +; CHECK: calll cdecl_void +; CHECK: retl $4 + +define x86_fastcallcc void @fastcall_cdecl_notail(i32 inreg %a, i32 inreg %b, i32 %c) { + tail call void @cdecl_void() + ret void +} +; CHECK-LABEL: fastcall_cdecl_notail +; CHECK: calll cdecl_void +; CHECK: retl $4 + + +; Tail call to/from callee pop functions can work under the right circumstances: + +declare x86_thiscallcc void @no_args_method(i8*) +declare x86_thiscallcc void @one_arg_method(i8*, i32) +declare x86_thiscallcc void @two_args_method(i8*, i32, i32) +declare void @ccall_func() +declare void @ccall_func1(i32) + +define x86_thiscallcc void @thiscall_thiscall_tail(i8* %this) { +entry: + tail call x86_thiscallcc void @no_args_method(i8* %this) + ret void +} +; CHECK-LABEL: thiscall_thiscall_tail: +; CHECK: jmp no_args_method + +define x86_thiscallcc void @thiscall_thiscall_tail2(i8* %this, i32 %a, i32 %b) { +entry: + tail call x86_thiscallcc void @two_args_method(i8* %this, i32 %a, i32 %b) + ret void +} +; @two_args_method will take care of popping %a and %b from the stack for us. +; CHECK-LABEL: thiscall_thiscall_tail2: +; CHECK: jmp two_args_method + +define x86_thiscallcc void @thiscall_thiscall_notail(i8* %this, i32 %a, i32 %b, i32 %x) { +entry: + tail call x86_thiscallcc void @two_args_method(i8* %this, i32 %a, i32 %b) + ret void +} +; @two_args_method would not pop %x. +; CHECK-LABEL: thiscall_thiscall_notail: +; CHECK: calll two_args_method +; CHECK: retl $12 + +define x86_thiscallcc void @thiscall_thiscall_notail2(i8* %this, i32 %a) { +entry: + tail call x86_thiscallcc void @no_args_method(i8* %this) + ret void +} +; @no_args_method would not pop %x for us. Make sure this is checked even +; when there are no arguments to the call. +; CHECK-LABEL: thiscall_thiscall_notail2: +; CHECK: calll no_args_method +; CHECK: retl $4 + +define void @ccall_thiscall_tail(i8* %x) { +entry: + tail call x86_thiscallcc void @no_args_method(i8* %x) + ret void +} +; Tail calling from ccall to thiscall works. +; CHECK-LABEL: ccall_thiscall_tail: +; CHECK: jmp no_args_method + +define void @ccall_thiscall_notail(i8* %x, i32 %y) { +entry: + tail call x86_thiscallcc void @one_arg_method(i8* %x, i32 %y); + ret void +} +; @one_arg_method would pop %y off the stack. +; CHECK-LABEL: ccall_thiscall_notail: +; CHECK: calll one_arg_method + +define x86_thiscallcc void @thiscall_ccall_tail(i8* %this) { +entry: + tail call void @ccall_func() + ret void +} +; Tail call from thiscall to ccall works if no arguments need popping. +; CHECK-LABEL: thiscall_ccall_tail: +; CHECK: jmp ccall_func + +define x86_thiscallcc void @thiscall_ccall_notail(i8* %this, i32 %x) { +entry: + tail call void @ccall_func1(i32 %x) + ret void +} +; No tail call: %x needs to be popped. +; CHECK-LABEL: thiscall_ccall_notail: +; CHECK: calll ccall_func1 +; CHECK: retl $4 + +%S = type { i32 (...)** } +define x86_thiscallcc void @tailcall_through_pointer(%S* %this, i32 %a) { +entry: + %0 = bitcast %S* %this to void (%S*, i32)*** + %vtable = load void (%S*, i32)**, void (%S*, i32)*** %0 + %1 = load void (%S*, i32)*, void (%S*, i32)** %vtable + tail call x86_thiscallcc void %1(%S* %this, i32 %a) + ret void +} +; Tail calling works through function pointers too. +; CHECK-LABEL: tailcall_through_pointer: +; CHECK: jmpl + +define x86_stdcallcc void @stdcall_cdecl_tail() { + tail call void @ccall_func() + ret void +} +; stdcall to cdecl works if no arguments need popping. +; CHECK-LABEL: stdcall_cdecl_tail +; CHECK: jmp ccall_func + +define x86_vectorcallcc void @vectorcall_cdecl_tail(i32 inreg %a, i32 inreg %b) { + tail call void @ccall_func() + ret void +} +; vectorcall to cdecl works if no arguments need popping. +; CHECK-LABEL: vectorcall_cdecl_tail +; CHECK: jmp ccall_func + +define x86_fastcallcc void @fastcall_cdecl_tail(i32 inreg %a, i32 inreg %b) { + tail call void @ccall_func() + ret void +} +; fastcall to cdecl works if no arguments need popping. +; CHECK-LABEL: fastcall_cdecl_tail +; CHECK: jmp ccall_func + +define x86_stdcallcc void @stdcall_thiscall_notail(i8* %this, i32 %a, i32 %b) { + tail call x86_thiscallcc void @two_args_method(i8* %this, i32 %a, i32 %b) + ret void +} +; two_args_method will not pop %this. +; CHECK-LABEL: stdcall_thiscall_notail +; CHECK: calll two_args_method + +define x86_stdcallcc void @stdcall_thiscall_tail(i32 %a, i32 %b) { + tail call x86_thiscallcc void @two_args_method(i8* null, i32 %a, i32 %b) + ret void +} +; The callee pop amounts match up. +; CHECK-LABEL: stdcall_thiscall_tail +; CHECK: jmp two_args_method + +declare x86_fastcallcc void @fastcall2(i32 inreg %a, i32 inreg %b) +define void @cdecl_fastcall_tail(i32 %a, i32 %b) { + tail call x86_fastcallcc void @fastcall2(i32 %a, i32 %b) + ret void +} +; fastcall2 won't pop anything. +; CHECK-LABEL: cdecl_fastcall_tail +; CHECK: jmp fastcall2 diff --git a/test/CodeGen/X86/tailcall-readnone.ll b/test/CodeGen/X86/tailcall-readnone.ll new file mode 100644 index 000000000000..b43f69120e7c --- /dev/null +++ b/test/CodeGen/X86/tailcall-readnone.ll @@ -0,0 +1,15 @@ +; RUN: llc -mtriple=x86_64-unknown-linux-gnu -o - %s | FileCheck %s + +define void @f(i32** %p) unnamed_addr { +entry: + %v = tail call i32* @g() + store i32* %v, i32** %p, align 8 + ret void +} +; CHECK-LABEL: f: +; CHECK: callq g +; CHECK: movq %rax, (%rbx) + +declare i32* @g() #2 + +attributes #2 = { nounwind readnone } diff --git a/test/CodeGen/X86/tls-android-negative.ll b/test/CodeGen/X86/tls-android-negative.ll new file mode 100644 index 000000000000..e90b8914ab28 --- /dev/null +++ b/test/CodeGen/X86/tls-android-negative.ll @@ -0,0 +1,65 @@ +; RUN: llc < %s -emulated-tls -march=x86 -mtriple=x86_64-linux-android -relocation-model=pic | FileCheck %s +; RUN: llc < %s -emulated-tls -march=x86-64 -mtriple=x86_64-linux-android -relocation-model=pic | FileCheck %s + +; Make sure that some symboles are not emitted in emulated TLS model. + +@external_x = external thread_local global i32 +@external_y = thread_local global i32 7 +@internal_y = internal thread_local global i32 9 +@internal_y0 = internal thread_local global i32 0 + +define i32* @get_external_x() { +entry: + ret i32* @external_x +} + +define i32* @get_external_y() { +entry: + ret i32* @external_y +} + +define i32* @get_internal_y() { +entry: + ret i32* @internal_y +} + +define i32* @get_internal_y0() { +entry: + ret i32* @internal_y0 +} + +; no direct access to emulated TLS variables. +; no definition of emulated TLS variables. +; no initializer for external TLS variables, __emutls_t.external_x +; no initializer for 0-initialized TLS variables, __emutls_t.internal_y0 +; not global linkage for __emutls_t.external_y + +; CHECK-NOT: external_x@TLS +; CHECK-NOT: external_y@TLS +; CHECK-NOT: internal_y@TLS +; CHECK-NOT: .size external_x +; CHECK-NOT: .size external_y +; CHECK-NOT: .size internal_y +; CHECK-NOT: .size internal_y0 +; CHECK-NOT: __emutls_v.external_x: +; CHECK-NOT: __emutls_t.external_x: +; CHECK-NOT: __emutls_t.internal_y0: +; CHECK-NOT: global __emutls_t.external_y +; CHECK-NOT: global __emutls_v.internal_y +; CHECK-NOT: global __emutls_v.internal_y0 + +; CHECK: __emutls_t.external_y + +; CHECK-NOT: external_x@TLS +; CHECK-NOT: external_y@TLS +; CHECK-NOT: internal_y@TLS +; CHECK-NOT: .size external_x +; CHECK-NOT: .size external_y +; CHECK-NOT: .size internal_y +; CHECK-NOT: .size internal_y0 +; CHECK-NOT: __emutls_v.external_x: +; CHECK-NOT: __emutls_t.external_x: +; CHECK-NOT: __emutls_t.internal_y0: +; CHECK-NOT: global __emutls_t.external_y +; CHECK-NOT: global __emutls_v.internal_y +; CHECK-NOT: global __emutls_v.internal_y0 diff --git a/test/CodeGen/X86/tls-android.ll b/test/CodeGen/X86/tls-android.ll new file mode 100644 index 000000000000..4156c7b3f5b9 --- /dev/null +++ b/test/CodeGen/X86/tls-android.ll @@ -0,0 +1,89 @@ +; RUN: llc < %s -emulated-tls -march=x86 -mtriple=x86_64-linux-android -relocation-model=pic | FileCheck %s +; RUN: llc < %s -emulated-tls -march=x86-64 -mtriple=x86_64-linux-android -relocation-model=pic | FileCheck -check-prefix=X64 %s + +; Make sure that TLS symboles are emitted in expected order. + +@external_x = external thread_local global i32 +@external_y = thread_local global i32 7 +@internal_y = internal thread_local global i32 9 + +define i32* @get_external_x() { +entry: + ret i32* @external_x +} + +define i32* @get_external_y() { +entry: + ret i32* @external_y +} + +define i32* @get_internal_y() { +entry: + ret i32* @internal_y +} + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 32-bit mode +; CHECK-LABEL: get_external_x: +; CHECK: __emutls_v.external_x +; CHECK: __emutls_get_address + +; CHECK-LABEL: get_external_y: +; CHECK: __emutls_v.external_y +; CHECK: __emutls_get_address + +; CHECK-LABEL: get_internal_y: +; CHECK: __emutls_v.internal_y +; CHECK: __emutls_get_address + +; CHECK-NOT: __emutls_v.external_x: + +; CHECK: .align 4 +; CHECK-LABEL: __emutls_v.external_y: +; CHECK-NEXT: .long 4 +; CHECK-NEXT: .long 4 +; CHECK-NEXT: .long 0 +; CHECK-NEXT: .long __emutls_t.external_y +; CHECK-LABEL: __emutls_t.external_y: +; CHECK-NEXT: .long 7 + +; CHECK: .align 4 +; CHECK-LABEL: __emutls_v.internal_y: +; CHECK-NEXT: .long 4 +; CHECK-NEXT: .long 4 +; CHECK-NEXT: .long 0 +; CHECK-NEXT: .long __emutls_t.internal_y +; CHECK-LABEL: __emutls_t.internal_y: +; CHECK-NEXT: .long 9 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 64-bit mode +; X64-LABEL: get_external_x: +; X64: __emutls_v.external_x +; X64: __emutls_get_address + +; X64-LABEL: get_external_y: +; X64: __emutls_v.external_y +; X64: __emutls_get_address + +; X64-LABEL: get_internal_y: +; X64: __emutls_v.internal_y +; X64: __emutls_get_address + +; X64-NOT: __emutls_v.external_x: + +; X64: .align 8 +; X64-LABEL: __emutls_v.external_y: +; X64-NEXT: .quad 4 +; X64-NEXT: .quad 4 +; X64-NEXT: .quad 0 +; X64-NEXT: .quad __emutls_t.external_y +; X64-LABEL: __emutls_t.external_y: +; X64-NEXT: .long 7 + +; X64: .align 8 +; X64-LABEL: __emutls_v.internal_y: +; X64-NEXT: .quad 4 +; X64-NEXT: .quad 4 +; X64-NEXT: .quad 0 +; X64-NEXT: .quad __emutls_t.internal_y +; X64-LABEL: __emutls_t.internal_y: +; X64-NEXT: .long 9 diff --git a/test/CodeGen/X86/tls-models.ll b/test/CodeGen/X86/tls-models.ll index 0fd785328211..2377da4f025a 100644 --- a/test/CodeGen/X86/tls-models.ll +++ b/test/CodeGen/X86/tls-models.ll @@ -18,6 +18,8 @@ @external_le = external thread_local(localexec) global i32 @internal_le = internal thread_local(localexec) global i32 42 +; See test cases for emulated model in emutls.ll, emutls-pic.ll and emutls-pie.ll. + ; ----- no model specified ----- define i32* @f1() { diff --git a/test/CodeGen/X86/tls-pie.ll b/test/CodeGen/X86/tls-pie.ll index 10fe1e94bbdc..235230e3c6a8 100644 --- a/test/CodeGen/X86/tls-pie.ll +++ b/test/CodeGen/X86/tls-pie.ll @@ -36,9 +36,13 @@ entry: define i32 @f3() { ; X32-LABEL: f3: ; X32: calll .L{{[0-9]+}}$pb +; X32-NEXT: .Ltmp{{[0-9]+}}: +; X32-NEXT: .cfi_adjust_cfa_offset 4 ; X32-NEXT: .L{{[0-9]+}}$pb: ; X32-NEXT: popl %eax ; X32-NEXT: .Ltmp{{[0-9]+}}: +; X32-NEXT: .cfi_adjust_cfa_offset -4 +; X32-NEXT: .Ltmp{{[0-9]+}}: ; X32-NEXT: addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp{{[0-9]+}}-.L{{[0-9]+}}$pb), %eax ; X32-NEXT: movl i2@GOTNTPOFF(%eax), %eax ; X32-NEXT: movl %gs:(%eax), %eax @@ -56,9 +60,13 @@ entry: define i32* @f4() { ; X32-LABEL: f4: ; X32: calll .L{{[0-9]+}}$pb +; X32-NEXT: .Ltmp{{[0-9]+}}: +; X32-NEXT: .cfi_adjust_cfa_offset 4 ; X32-NEXT: .L{{[0-9]+}}$pb: ; X32-NEXT: popl %ecx ; X32-NEXT: .Ltmp{{[0-9]+}}: +; X32-NEXT: .cfi_adjust_cfa_offset -4 +; X32-NEXT: .Ltmp{{[0-9]+}}: ; X32-NEXT: addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp{{[0-9]+}}-.L{{[0-9]+}}$pb), %ecx ; X32-NEXT: movl %gs:0, %eax ; X32-NEXT: addl i2@GOTNTPOFF(%ecx), %eax diff --git a/test/CodeGen/X86/token_landingpad.ll b/test/CodeGen/X86/token_landingpad.ll new file mode 100644 index 000000000000..087b68bfce8a --- /dev/null +++ b/test/CodeGen/X86/token_landingpad.ll @@ -0,0 +1,21 @@ +; RUN: llc < %s + +; This test verifies that SelectionDAG can handle landingPad of token type and not crash LLVM. + +define void @test() personality i32 (...)* @dummy_personality { +entry: + invoke void @dummy() + to label %return unwind label %unwind + +unwind: ; preds = %entry + %lp = landingpad token + cleanup + br label %return + +return: ; preds = %entry + ret void +} + +declare void @dummy() + +declare i32 @dummy_personality(...) diff --git a/test/CodeGen/X86/trunc-store.ll b/test/CodeGen/X86/trunc-store.ll new file mode 100644 index 000000000000..646b4b2c336d --- /dev/null +++ b/test/CodeGen/X86/trunc-store.ll @@ -0,0 +1,49 @@ +; RUN: llc < %s -march=x86-64 | FileCheck %s + +; With optimization at O2 we actually get the legalized function optimized +; away through legalization and stack coloring, but check that we do all of +; that here and don't crash during legalization. + +; Original program: +; typedef enum { A, B, C, D } P; +; struct { P x[2]; } a; + +; void fn2(); +; void fn1() { +; int b; +; unsigned c; +; for (;; c++) { +; fn2(); +; unsigned n; +; for (; c; c++) { +; b = a.x[c] == A || a.x[c] == B || a.x[c] == D; +; if (b) n++; +; } +; if (n) +; for (;;) +; ; +; } +; } + +define void @fn1() { +; CHECK-LABEL: fn1 +; CHECK: movb $0, {{.*}}(%rsp) +; CHECK: cmpq $8, %rax +for.cond: + br label %vector.body + +vector.body: ; preds = %vector.body, %for.cond + %x42 = bitcast <4 x i4> zeroinitializer to i16 + %x43 = icmp ne i16 %x42, 0 + %x44 = select i1 %x43, i32 undef, i32 0 + %x72 = bitcast <4 x i1> zeroinitializer to i4 + %x73 = icmp ne i4 %x72, 0 + %x74 = select i1 %x73, i32 %x44, i32 undef + %x84 = select i1 undef, i32 undef, i32 %x74 + %x88 = icmp eq i64 undef, 8 + br i1 %x88, label %middle.block, label %vector.body + +middle.block: ; preds = %vector.body + %0 = select i1 undef, i32 undef, i32 %x84 + ret void +} diff --git a/test/CodeGen/X86/unaligned-32-byte-memops.ll b/test/CodeGen/X86/unaligned-32-byte-memops.ll index d979c16f4abd..b9deb058cb3f 100644 --- a/test/CodeGen/X86/unaligned-32-byte-memops.ll +++ b/test/CodeGen/X86/unaligned-32-byte-memops.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx,+slow-unaligned-mem-32 | FileCheck %s --check-prefix=AVXSLOW ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx,-slow-unaligned-mem-32 | FileCheck %s --check-prefix=AVXFAST ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2 | FileCheck %s --check-prefix=AVX2 @@ -75,12 +76,12 @@ define <8 x float> @combine_16_byte_loads_no_intrinsic(<4 x float>* %ptr) { ret <8 x float> %v3 } +; If the first load is 32-byte aligned, then the loads should be merged in all cases. + define <8 x float> @combine_16_byte_loads_aligned(<4 x float>* %ptr) { -;; FIXME: The first load is 32-byte aligned, so the second load should get merged. ; AVXSLOW-LABEL: combine_16_byte_loads_aligned: ; AVXSLOW: # BB#0: -; AVXSLOW-NEXT: vmovaps 48(%rdi), %xmm0 -; AVXSLOW-NEXT: vinsertf128 $1, 64(%rdi), %ymm0, %ymm0 +; AVXSLOW-NEXT: vmovaps 48(%rdi), %ymm0 ; AVXSLOW-NEXT: retq ; ; AVXFAST-LABEL: combine_16_byte_loads_aligned: diff --git a/test/CodeGen/X86/unaligned-spill-folding.ll b/test/CodeGen/X86/unaligned-spill-folding.ll index 33e2daf9dc1b..dee94bce15a5 100644 --- a/test/CodeGen/X86/unaligned-spill-folding.ll +++ b/test/CodeGen/X86/unaligned-spill-folding.ll @@ -1,6 +1,6 @@ ; RUN: llc -mtriple=i386-unknown-freebsd -mcpu=core2 -stack-alignment=4 -relocation-model=pic < %s | FileCheck %s -check-prefix=UNALIGNED ; RUN: llc -mtriple=i386-unknown-freebsd -mcpu=core2 -stack-alignment=16 -relocation-model=pic < %s | FileCheck %s -check-prefix=ALIGNED -; RUN: llc -mtriple=i386-unknown-freebsd -mcpu=core2 -stack-alignment=4 -force-align-stack -relocation-model=pic < %s | FileCheck %s -check-prefix=FORCEALIGNED +; RUN: llc -mtriple=i386-unknown-freebsd -mcpu=core2 -stack-alignment=4 -stackrealign -relocation-model=pic < %s | FileCheck %s -check-prefix=FORCEALIGNED @arr = internal unnamed_addr global [32 x i32] zeroinitializer, align 16 diff --git a/test/CodeGen/X86/unknown-location.ll b/test/CodeGen/X86/unknown-location.ll index c018a49d135e..c41e529aa954 100644 --- a/test/CodeGen/X86/unknown-location.ll +++ b/test/CodeGen/X86/unknown-location.ll @@ -10,7 +10,7 @@ ; CHECK-NEXT: idivl ; CHECK-NEXT: .loc 1 4 3 -define i32 @foo(i32 %w, i32 %x, i32 %y, i32 %z) nounwind { +define i32 @foo(i32 %w, i32 %x, i32 %y, i32 %z) nounwind !dbg !1 { entry: %a = add i32 %w, %x, !dbg !8 %b = sdiv i32 %a, %y @@ -21,10 +21,10 @@ entry: !llvm.dbg.cu = !{!3} !llvm.module.flags = !{!12} -!0 = !DILocalVariable(tag: DW_TAG_arg_variable, name: "x", line: 1, arg: 0, scope: !1, file: !2, type: !6) -!1 = !DISubprogram(name: "foo", linkageName: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !10, scope: !2, type: !4, function: i32 (i32, i32, i32, i32)* @foo) +!0 = !DILocalVariable(name: "x", line: 1, arg: 2, scope: !1, file: !2, type: !6) +!1 = distinct !DISubprogram(name: "foo", linkageName: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, scopeLine: 1, file: !10, scope: !2, type: !4) !2 = !DIFile(filename: "test.c", directory: "/dir") -!3 = !DICompileUnit(language: DW_LANG_C99, producer: "producer", isOptimized: false, emissionKind: 0, file: !10, enums: !11, retainedTypes: !11, subprograms: !9) +!3 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "producer", isOptimized: false, emissionKind: 0, file: !10, enums: !11, retainedTypes: !11, subprograms: !9) !4 = !DISubroutineType(types: !5) !5 = !{!6} !6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) diff --git a/test/CodeGen/X86/v2f32.ll b/test/CodeGen/X86/v2f32.ll index 7beed52295ee..3b7160c71869 100644 --- a/test/CodeGen/X86/v2f32.ll +++ b/test/CodeGen/X86/v2f32.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-linux -mcpu=penryn -o - | FileCheck %s --check-prefix=X64 ; RUN: llc < %s -mcpu=yonah -march=x86 -mtriple=i386-linux-gnu -o - | FileCheck %s --check-prefix=X32 diff --git a/test/CodeGen/X86/vec_cast2.ll b/test/CodeGen/X86/vec_cast2.ll index 1ba11f51baa2..dda50b7b94b7 100644 --- a/test/CodeGen/X86/vec_cast2.ll +++ b/test/CodeGen/X86/vec_cast2.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=i386-apple-darwin10 -mcpu=corei7-avx -mattr=+avx | FileCheck %s ; RUN: llc < %s -mtriple=i386-apple-darwin10 -mcpu=corei7-avx -mattr=+avx -x86-experimental-vector-widening-legalization | FileCheck %s --check-prefix=CHECK-WIDE @@ -46,17 +47,19 @@ define <4 x float> @foo1_4(<4 x i8> %src) { define <8 x float> @foo2_8(<8 x i8> %src) { ; CHECK-LABEL: foo2_8: ; CHECK: ## BB#0: -; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero -; CHECK-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7] -; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 -; CHECK-NEXT: vandps LCPI2_0, %ymm0, %ymm0 +; CHECK-NEXT: vpand LCPI2_0, %xmm0, %xmm0 +; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; CHECK-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; CHECK-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; CHECK-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; CHECK-NEXT: vcvtdq2ps %ymm0, %ymm0 ; CHECK-NEXT: retl ; ; CHECK-WIDE-LABEL: foo2_8: ; CHECK-WIDE: ## BB#0: ; CHECK-WIDE-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero -; CHECK-WIDE-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero +; CHECK-WIDE-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; CHECK-WIDE-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero ; CHECK-WIDE-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 ; CHECK-WIDE-NEXT: vcvtdq2ps %ymm0, %ymm0 ; CHECK-WIDE-NEXT: retl @@ -96,25 +99,25 @@ define <8 x i8> @foo3_8(<8 x float> %src) { ; CHECK-WIDE: ## BB#0: ; CHECK-WIDE-NEXT: vcvttss2si %xmm0, %eax ; CHECK-WIDE-NEXT: vpinsrb $0, %eax, %xmm0, %xmm1 -; CHECK-WIDE-NEXT: vmovshdup %xmm0, %xmm2 ## xmm2 = xmm0[1,1,3,3] +; CHECK-WIDE-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3] ; CHECK-WIDE-NEXT: vcvttss2si %xmm2, %eax ; CHECK-WIDE-NEXT: vpinsrb $1, %eax, %xmm1, %xmm1 -; CHECK-WIDE-NEXT: vpermilpd $1, %xmm0, %xmm2 ## xmm2 = xmm0[1,0] +; CHECK-WIDE-NEXT: vpermilpd {{.*#+}} xmm2 = xmm0[1,0] ; CHECK-WIDE-NEXT: vcvttss2si %xmm2, %eax ; CHECK-WIDE-NEXT: vpinsrb $2, %eax, %xmm1, %xmm1 -; CHECK-WIDE-NEXT: vpermilps $231, %xmm0, %xmm2 ## xmm2 = xmm0[3,1,2,3] +; CHECK-WIDE-NEXT: vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3] ; CHECK-WIDE-NEXT: vcvttss2si %xmm2, %eax ; CHECK-WIDE-NEXT: vpinsrb $3, %eax, %xmm1, %xmm1 ; CHECK-WIDE-NEXT: vextractf128 $1, %ymm0, %xmm0 ; CHECK-WIDE-NEXT: vcvttss2si %xmm0, %eax ; CHECK-WIDE-NEXT: vpinsrb $4, %eax, %xmm1, %xmm1 -; CHECK-WIDE-NEXT: vmovshdup %xmm0, %xmm2 ## xmm2 = xmm0[1,1,3,3] +; CHECK-WIDE-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3] ; CHECK-WIDE-NEXT: vcvttss2si %xmm2, %eax ; CHECK-WIDE-NEXT: vpinsrb $5, %eax, %xmm1, %xmm1 -; CHECK-WIDE-NEXT: vpermilpd $1, %xmm0, %xmm2 ## xmm2 = xmm0[1,0] +; CHECK-WIDE-NEXT: vpermilpd {{.*#+}} xmm2 = xmm0[1,0] ; CHECK-WIDE-NEXT: vcvttss2si %xmm2, %eax ; CHECK-WIDE-NEXT: vpinsrb $6, %eax, %xmm1, %xmm1 -; CHECK-WIDE-NEXT: vpermilps $231, %xmm0, %xmm0 ## xmm0 = xmm0[3,1,2,3] +; CHECK-WIDE-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3] ; CHECK-WIDE-NEXT: vcvttss2si %xmm0, %eax ; CHECK-WIDE-NEXT: vpinsrb $7, %eax, %xmm1, %xmm0 ; CHECK-WIDE-NEXT: vzeroupper @@ -133,13 +136,13 @@ define <4 x i8> @foo3_4(<4 x float> %src) { ; CHECK-WIDE: ## BB#0: ; CHECK-WIDE-NEXT: vcvttss2si %xmm0, %eax ; CHECK-WIDE-NEXT: vpinsrb $0, %eax, %xmm0, %xmm1 -; CHECK-WIDE-NEXT: vmovshdup %xmm0, %xmm2 ## xmm2 = xmm0[1,1,3,3] +; CHECK-WIDE-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3] ; CHECK-WIDE-NEXT: vcvttss2si %xmm2, %eax ; CHECK-WIDE-NEXT: vpinsrb $1, %eax, %xmm1, %xmm1 -; CHECK-WIDE-NEXT: vpermilpd $1, %xmm0, %xmm2 ## xmm2 = xmm0[1,0] +; CHECK-WIDE-NEXT: vpermilpd {{.*#+}} xmm2 = xmm0[1,0] ; CHECK-WIDE-NEXT: vcvttss2si %xmm2, %eax ; CHECK-WIDE-NEXT: vpinsrb $2, %eax, %xmm1, %xmm1 -; CHECK-WIDE-NEXT: vpermilps $231, %xmm0, %xmm0 ## xmm0 = xmm0[3,1,2,3] +; CHECK-WIDE-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3] ; CHECK-WIDE-NEXT: vcvttss2si %xmm0, %eax ; CHECK-WIDE-NEXT: vpinsrb $3, %eax, %xmm1, %xmm0 ; CHECK-WIDE-NEXT: retl diff --git a/test/CodeGen/X86/vec_cmp_sint-128.ll b/test/CodeGen/X86/vec_cmp_sint-128.ll new file mode 100644 index 000000000000..1407f71de714 --- /dev/null +++ b/test/CodeGen/X86/vec_cmp_sint-128.ll @@ -0,0 +1,722 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE42 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+xop | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+xop | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW + +; +; Equal +; + +define <2 x i64> @eq_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { +; SSE2-LABEL: eq_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2] +; SSE2-NEXT: pand %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: eq_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: pcmpeqq %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: eq_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: pcmpeqq %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: eq_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpcmpeqq %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: eq_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpcomeqq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp eq <2 x i64> %a, %b + %2 = sext <2 x i1> %1 to <2 x i64> + ret <2 x i64> %2 +} + +define <4 x i32> @eq_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { +; SSE-LABEL: eq_v4i32: +; SSE: # BB#0: +; SSE-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: eq_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: eq_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpcomeqd %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp eq <4 x i32> %a, %b + %2 = sext <4 x i1> %1 to <4 x i32> + ret <4 x i32> %2 +} + +define <8 x i16> @eq_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { +; SSE-LABEL: eq_v8i16: +; SSE: # BB#0: +; SSE-NEXT: pcmpeqw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: eq_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: eq_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpcomeqw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp eq <8 x i16> %a, %b + %2 = sext <8 x i1> %1 to <8 x i16> + ret <8 x i16> %2 +} + +define <16 x i8> @eq_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { +; SSE-LABEL: eq_v16i8: +; SSE: # BB#0: +; SSE-NEXT: pcmpeqb %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: eq_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: eq_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpcomeqb %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp eq <16 x i8> %a, %b + %2 = sext <16 x i1> %1 to <16 x i8> + ret <16 x i8> %2 +} + +; +; Not Equal +; + +define <2 x i64> @ne_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { +; SSE2-LABEL: ne_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2] +; SSE2-NEXT: pand %xmm1, %xmm0 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: pxor %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: ne_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: pcmpeqq %xmm1, %xmm0 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE41-NEXT: pxor %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: ne_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: pcmpeqq %xmm1, %xmm0 +; SSE42-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE42-NEXT: pxor %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: ne_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpcmpeqq %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: ne_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpcomneqq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ne <2 x i64> %a, %b + %2 = sext <2 x i1> %1 to <2 x i64> + ret <2 x i64> %2 +} + +define <4 x i32> @ne_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { +; SSE-LABEL: ne_v4i32: +; SSE: # BB#0: +; SSE-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE-NEXT: pxor %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ne_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: ne_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpcomneqd %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ne <4 x i32> %a, %b + %2 = sext <4 x i1> %1 to <4 x i32> + ret <4 x i32> %2 +} + +define <8 x i16> @ne_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { +; SSE-LABEL: ne_v8i16: +; SSE: # BB#0: +; SSE-NEXT: pcmpeqw %xmm1, %xmm0 +; SSE-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE-NEXT: pxor %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ne_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: ne_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpcomneqw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ne <8 x i16> %a, %b + %2 = sext <8 x i1> %1 to <8 x i16> + ret <8 x i16> %2 +} + +define <16 x i8> @ne_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { +; SSE-LABEL: ne_v16i8: +; SSE: # BB#0: +; SSE-NEXT: pcmpeqb %xmm1, %xmm0 +; SSE-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE-NEXT: pxor %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ne_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: ne_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpcomneqb %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ne <16 x i8> %a, %b + %2 = sext <16 x i1> %1 to <16 x i8> + ret <16 x i8> %2 +} + +; +; Greater Than Or Equal +; + +define <2 x i64> @ge_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { +; SSE2-LABEL: ge_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE2-NEXT: pxor %xmm2, %xmm0 +; SSE2-NEXT: pxor %xmm2, %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm1 +; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE2-NEXT: pxor %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: ge_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] +; SSE41-NEXT: pand %xmm3, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3] +; SSE41-NEXT: por %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE41-NEXT: pxor %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: ge_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: pcmpgtq %xmm0, %xmm1 +; SSE42-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE42-NEXT: pxor %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: ge_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: ge_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpcomgeq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp sge <2 x i64> %a, %b + %2 = sext <2 x i1> %1 to <2 x i64> + ret <2 x i64> %2 +} + +define <4 x i32> @ge_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { +; SSE-LABEL: ge_v4i32: +; SSE: # BB#0: +; SSE-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE-NEXT: pxor %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ge_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtd %xmm0, %xmm1, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: ge_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpcomged %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp sge <4 x i32> %a, %b + %2 = sext <4 x i1> %1 to <4 x i32> + ret <4 x i32> %2 +} + +define <8 x i16> @ge_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { +; SSE-LABEL: ge_v8i16: +; SSE: # BB#0: +; SSE-NEXT: pcmpgtw %xmm0, %xmm1 +; SSE-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE-NEXT: pxor %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ge_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtw %xmm0, %xmm1, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: ge_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpcomgew %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp sge <8 x i16> %a, %b + %2 = sext <8 x i1> %1 to <8 x i16> + ret <8 x i16> %2 +} + +define <16 x i8> @ge_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { +; SSE-LABEL: ge_v16i8: +; SSE: # BB#0: +; SSE-NEXT: pcmpgtb %xmm0, %xmm1 +; SSE-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE-NEXT: pxor %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ge_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtb %xmm0, %xmm1, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: ge_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpcomgeb %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp sge <16 x i8> %a, %b + %2 = sext <16 x i1> %1 to <16 x i8> + ret <16 x i8> %2 +} + +; +; Greater Than +; + +define <2 x i64> @gt_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { +; SSE2-LABEL: gt_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE2-NEXT: pxor %xmm2, %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm0 +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] +; SSE2-NEXT: pand %xmm3, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE2-NEXT: por %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: gt_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE41-NEXT: pxor %xmm2, %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm3, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: por %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: gt_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: pcmpgtq %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: gt_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: gt_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpcomgtq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp sgt <2 x i64> %a, %b + %2 = sext <2 x i1> %1 to <2 x i64> + ret <2 x i64> %2 +} + +define <4 x i32> @gt_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { +; SSE-LABEL: gt_v4i32: +; SSE: # BB#0: +; SSE-NEXT: pcmpgtd %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: gt_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: gt_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpcomgtd %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp sgt <4 x i32> %a, %b + %2 = sext <4 x i1> %1 to <4 x i32> + ret <4 x i32> %2 +} + +define <8 x i16> @gt_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { +; SSE-LABEL: gt_v8i16: +; SSE: # BB#0: +; SSE-NEXT: pcmpgtw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: gt_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: gt_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpcomgtw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp sgt <8 x i16> %a, %b + %2 = sext <8 x i1> %1 to <8 x i16> + ret <8 x i16> %2 +} + +define <16 x i8> @gt_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { +; SSE-LABEL: gt_v16i8: +; SSE: # BB#0: +; SSE-NEXT: pcmpgtb %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: gt_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: gt_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpcomgtb %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp sgt <16 x i8> %a, %b + %2 = sext <16 x i1> %1 to <16 x i8> + ret <16 x i8> %2 +} + +; +; Less Than Or Equal +; + +define <2 x i64> @le_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { +; SSE2-LABEL: le_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE2-NEXT: pxor %xmm2, %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm0 +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm1 +; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE2-NEXT: pxor %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: le_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE41-NEXT: pxor %xmm2, %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm3, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3] +; SSE41-NEXT: por %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE41-NEXT: pxor %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: le_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: pcmpgtq %xmm1, %xmm0 +; SSE42-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE42-NEXT: pxor %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: le_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: le_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpcomleq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp sle <2 x i64> %a, %b + %2 = sext <2 x i1> %1 to <2 x i64> + ret <2 x i64> %2 +} + +define <4 x i32> @le_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { +; SSE-LABEL: le_v4i32: +; SSE: # BB#0: +; SSE-NEXT: pcmpgtd %xmm1, %xmm0 +; SSE-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE-NEXT: pxor %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: le_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: le_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpcomled %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp sle <4 x i32> %a, %b + %2 = sext <4 x i1> %1 to <4 x i32> + ret <4 x i32> %2 +} + +define <8 x i16> @le_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { +; SSE-LABEL: le_v8i16: +; SSE: # BB#0: +; SSE-NEXT: pcmpgtw %xmm1, %xmm0 +; SSE-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE-NEXT: pxor %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: le_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: le_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpcomlew %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp sle <8 x i16> %a, %b + %2 = sext <8 x i1> %1 to <8 x i16> + ret <8 x i16> %2 +} + +define <16 x i8> @le_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { +; SSE-LABEL: le_v16i8: +; SSE: # BB#0: +; SSE-NEXT: pcmpgtb %xmm1, %xmm0 +; SSE-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE-NEXT: pxor %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: le_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: le_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpcomleb %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp sle <16 x i8> %a, %b + %2 = sext <16 x i1> %1 to <16 x i8> + ret <16 x i8> %2 +} + +; +; Less Than +; + +define <2 x i64> @lt_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { +; SSE2-LABEL: lt_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE2-NEXT: pxor %xmm2, %xmm0 +; SSE2-NEXT: pxor %xmm2, %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; SSE2-NEXT: pand %xmm3, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE2-NEXT: por %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: lt_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; SSE41-NEXT: pand %xmm3, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: por %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: lt_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: pcmpgtq %xmm0, %xmm1 +; SSE42-NEXT: movdqa %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: lt_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: lt_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpcomltq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp slt <2 x i64> %a, %b + %2 = sext <2 x i1> %1 to <2 x i64> + ret <2 x i64> %2 +} + +define <4 x i32> @lt_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { +; SSE-LABEL: lt_v4i32: +; SSE: # BB#0: +; SSE-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: lt_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtd %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: lt_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpcomltd %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp slt <4 x i32> %a, %b + %2 = sext <4 x i1> %1 to <4 x i32> + ret <4 x i32> %2 +} + +define <8 x i16> @lt_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { +; SSE-LABEL: lt_v8i16: +; SSE: # BB#0: +; SSE-NEXT: pcmpgtw %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: lt_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtw %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: lt_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpcomltw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp slt <8 x i16> %a, %b + %2 = sext <8 x i1> %1 to <8 x i16> + ret <8 x i16> %2 +} + +define <16 x i8> @lt_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { +; SSE-LABEL: lt_v16i8: +; SSE: # BB#0: +; SSE-NEXT: pcmpgtb %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: lt_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtb %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: lt_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpcomltb %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp slt <16 x i8> %a, %b + %2 = sext <16 x i1> %1 to <16 x i8> + ret <16 x i8> %2 +} diff --git a/test/CodeGen/X86/vec_cmp_uint-128.ll b/test/CodeGen/X86/vec_cmp_uint-128.ll new file mode 100644 index 000000000000..8bed14e7e5f5 --- /dev/null +++ b/test/CodeGen/X86/vec_cmp_uint-128.ll @@ -0,0 +1,860 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE42 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+xop | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+xop | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW + +; +; Equal +; + +define <2 x i64> @eq_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { +; SSE2-LABEL: eq_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2] +; SSE2-NEXT: pand %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: eq_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: pcmpeqq %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: eq_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: pcmpeqq %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: eq_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpcmpeqq %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: eq_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpcomeqq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp eq <2 x i64> %a, %b + %2 = sext <2 x i1> %1 to <2 x i64> + ret <2 x i64> %2 +} + +define <4 x i32> @eq_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { +; SSE-LABEL: eq_v4i32: +; SSE: # BB#0: +; SSE-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: eq_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: eq_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpcomeqd %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp eq <4 x i32> %a, %b + %2 = sext <4 x i1> %1 to <4 x i32> + ret <4 x i32> %2 +} + +define <8 x i16> @eq_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { +; SSE-LABEL: eq_v8i16: +; SSE: # BB#0: +; SSE-NEXT: pcmpeqw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: eq_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: eq_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpcomeqw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp eq <8 x i16> %a, %b + %2 = sext <8 x i1> %1 to <8 x i16> + ret <8 x i16> %2 +} + +define <16 x i8> @eq_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { +; SSE-LABEL: eq_v16i8: +; SSE: # BB#0: +; SSE-NEXT: pcmpeqb %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: eq_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: eq_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpcomeqb %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp eq <16 x i8> %a, %b + %2 = sext <16 x i1> %1 to <16 x i8> + ret <16 x i8> %2 +} + +; +; Not Equal +; + +define <2 x i64> @ne_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { +; SSE2-LABEL: ne_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2] +; SSE2-NEXT: pand %xmm1, %xmm0 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: pxor %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: ne_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: pcmpeqq %xmm1, %xmm0 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE41-NEXT: pxor %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: ne_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: pcmpeqq %xmm1, %xmm0 +; SSE42-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE42-NEXT: pxor %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: ne_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpcmpeqq %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: ne_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpcomneqq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ne <2 x i64> %a, %b + %2 = sext <2 x i1> %1 to <2 x i64> + ret <2 x i64> %2 +} + +define <4 x i32> @ne_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { +; SSE-LABEL: ne_v4i32: +; SSE: # BB#0: +; SSE-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE-NEXT: pxor %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ne_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: ne_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpcomneqd %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ne <4 x i32> %a, %b + %2 = sext <4 x i1> %1 to <4 x i32> + ret <4 x i32> %2 +} + +define <8 x i16> @ne_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { +; SSE-LABEL: ne_v8i16: +; SSE: # BB#0: +; SSE-NEXT: pcmpeqw %xmm1, %xmm0 +; SSE-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE-NEXT: pxor %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ne_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: ne_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpcomneqw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ne <8 x i16> %a, %b + %2 = sext <8 x i1> %1 to <8 x i16> + ret <8 x i16> %2 +} + +define <16 x i8> @ne_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { +; SSE-LABEL: ne_v16i8: +; SSE: # BB#0: +; SSE-NEXT: pcmpeqb %xmm1, %xmm0 +; SSE-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE-NEXT: pxor %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ne_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: ne_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpcomneqb %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ne <16 x i8> %a, %b + %2 = sext <16 x i1> %1 to <16 x i8> + ret <16 x i8> %2 +} + +; +; Greater Than Or Equal +; + +define <2 x i64> @ge_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { +; SSE2-LABEL: ge_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: pxor %xmm2, %xmm0 +; SSE2-NEXT: pxor %xmm2, %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm1 +; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE2-NEXT: pxor %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: ge_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] +; SSE41-NEXT: pand %xmm3, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3] +; SSE41-NEXT: por %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE41-NEXT: pxor %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: ge_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; SSE42-NEXT: pxor %xmm2, %xmm0 +; SSE42-NEXT: pxor %xmm1, %xmm2 +; SSE42-NEXT: pcmpgtq %xmm0, %xmm2 +; SSE42-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE42-NEXT: pxor %xmm2, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: ge_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: ge_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpcomgeuq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp uge <2 x i64> %a, %b + %2 = sext <2 x i1> %1 to <2 x i64> + ret <2 x i64> %2 +} + +define <4 x i32> @ge_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { +; SSE2-LABEL: ge_v4i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: pxor %xmm2, %xmm0 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE2-NEXT: pxor %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: ge_v4i32: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxud %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: ge_v4i32: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxud %xmm0, %xmm1 +; SSE42-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: ge_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpmaxud %xmm1, %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: ge_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpcomgeud %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp uge <4 x i32> %a, %b + %2 = sext <4 x i1> %1 to <4 x i32> + ret <4 x i32> %2 +} + +define <8 x i16> @ge_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { +; SSE2-LABEL: ge_v8i16: +; SSE2: # BB#0: +; SSE2-NEXT: psubusw %xmm0, %xmm1 +; SSE2-NEXT: pxor %xmm0, %xmm0 +; SSE2-NEXT: pcmpeqw %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: ge_v8i16: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxuw %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqw %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: ge_v8i16: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxuw %xmm0, %xmm1 +; SSE42-NEXT: pcmpeqw %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: ge_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpmaxuw %xmm1, %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: ge_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpcomgeuw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp uge <8 x i16> %a, %b + %2 = sext <8 x i1> %1 to <8 x i16> + ret <8 x i16> %2 +} + +define <16 x i8> @ge_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { +; SSE-LABEL: ge_v16i8: +; SSE: # BB#0: +; SSE-NEXT: pmaxub %xmm0, %xmm1 +; SSE-NEXT: pcmpeqb %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ge_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpmaxub %xmm1, %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: ge_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpcomgeub %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp uge <16 x i8> %a, %b + %2 = sext <16 x i1> %1 to <16 x i8> + ret <16 x i8> %2 +} + +; +; Greater Than +; + +define <2 x i64> @gt_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { +; SSE2-LABEL: gt_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: pxor %xmm2, %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm0 +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] +; SSE2-NEXT: pand %xmm3, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE2-NEXT: por %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: gt_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE41-NEXT: pxor %xmm2, %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm3, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: por %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: gt_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; SSE42-NEXT: pxor %xmm2, %xmm1 +; SSE42-NEXT: pxor %xmm2, %xmm0 +; SSE42-NEXT: pcmpgtq %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: gt_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: gt_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpcomgtuq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ugt <2 x i64> %a, %b + %2 = sext <2 x i1> %1 to <2 x i64> + ret <2 x i64> %2 +} + +define <4 x i32> @gt_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { +; SSE-LABEL: gt_v4i32: +; SSE: # BB#0: +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE-NEXT: pxor %xmm2, %xmm1 +; SSE-NEXT: pxor %xmm2, %xmm0 +; SSE-NEXT: pcmpgtd %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: gt_v4i32: +; AVX1: # BB#0: +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpcmpgtd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: gt_v4i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %xmm2 +; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpcmpgtd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; XOP-LABEL: gt_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpcomgtud %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: gt_v4i32: +; AVX512: # BB#0: +; AVX512-NEXT: vpbroadcastd {{.*}}(%rip), %xmm2 +; AVX512-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX512-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vpcmpgtd %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq + %1 = icmp ugt <4 x i32> %a, %b + %2 = sext <4 x i1> %1 to <4 x i32> + ret <4 x i32> %2 +} + +define <8 x i16> @gt_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { +; SSE-LABEL: gt_v8i16: +; SSE: # BB#0: +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE-NEXT: pxor %xmm2, %xmm1 +; SSE-NEXT: pxor %xmm2, %xmm0 +; SSE-NEXT: pcmpgtw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: gt_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768] +; AVX-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpcmpgtw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: gt_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpcomgtuw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ugt <8 x i16> %a, %b + %2 = sext <8 x i1> %1 to <8 x i16> + ret <8 x i16> %2 +} + +define <16 x i8> @gt_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { +; SSE-LABEL: gt_v16i8: +; SSE: # BB#0: +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128] +; SSE-NEXT: pxor %xmm2, %xmm1 +; SSE-NEXT: pxor %xmm2, %xmm0 +; SSE-NEXT: pcmpgtb %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: gt_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128] +; AVX-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpcmpgtb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: gt_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpcomgtub %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ugt <16 x i8> %a, %b + %2 = sext <16 x i1> %1 to <16 x i8> + ret <16 x i8> %2 +} + +; +; Less Than Or Equal +; + +define <2 x i64> @le_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { +; SSE2-LABEL: le_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: pxor %xmm2, %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm0 +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm1 +; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE2-NEXT: pxor %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: le_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE41-NEXT: pxor %xmm2, %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm3, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3] +; SSE41-NEXT: por %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE41-NEXT: pxor %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: le_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; SSE42-NEXT: pxor %xmm2, %xmm1 +; SSE42-NEXT: pxor %xmm2, %xmm0 +; SSE42-NEXT: pcmpgtq %xmm1, %xmm0 +; SSE42-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE42-NEXT: pxor %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: le_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: le_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpcomleuq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ule <2 x i64> %a, %b + %2 = sext <2 x i1> %1 to <2 x i64> + ret <2 x i64> %2 +} + +define <4 x i32> @le_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { +; SSE2-LABEL: le_v4i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: pxor %xmm2, %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm0 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm0 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: pxor %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: le_v4i32: +; SSE41: # BB#0: +; SSE41-NEXT: pminud %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: le_v4i32: +; SSE42: # BB#0: +; SSE42-NEXT: pminud %xmm0, %xmm1 +; SSE42-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: le_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpminud %xmm1, %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: le_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpcomleud %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ule <4 x i32> %a, %b + %2 = sext <4 x i1> %1 to <4 x i32> + ret <4 x i32> %2 +} + +define <8 x i16> @le_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { +; SSE2-LABEL: le_v8i16: +; SSE2: # BB#0: +; SSE2-NEXT: psubusw %xmm1, %xmm0 +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: pcmpeqw %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: le_v8i16: +; SSE41: # BB#0: +; SSE41-NEXT: pminuw %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqw %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: le_v8i16: +; SSE42: # BB#0: +; SSE42-NEXT: pminuw %xmm0, %xmm1 +; SSE42-NEXT: pcmpeqw %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: le_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpminuw %xmm1, %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: le_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpcomleuw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ule <8 x i16> %a, %b + %2 = sext <8 x i1> %1 to <8 x i16> + ret <8 x i16> %2 +} + +define <16 x i8> @le_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { +; SSE-LABEL: le_v16i8: +; SSE: # BB#0: +; SSE-NEXT: pminub %xmm0, %xmm1 +; SSE-NEXT: pcmpeqb %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: le_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpminub %xmm1, %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: le_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpcomleub %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ule <16 x i8> %a, %b + %2 = sext <16 x i1> %1 to <16 x i8> + ret <16 x i8> %2 +} + +; +; Less Than +; + +define <2 x i64> @lt_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { +; SSE2-LABEL: lt_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: pxor %xmm2, %xmm0 +; SSE2-NEXT: pxor %xmm2, %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; SSE2-NEXT: pand %xmm3, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE2-NEXT: por %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: lt_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: pxor %xmm2, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm2 +; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; SSE41-NEXT: pand %xmm3, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: por %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: lt_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; SSE42-NEXT: pxor %xmm2, %xmm0 +; SSE42-NEXT: pxor %xmm1, %xmm2 +; SSE42-NEXT: pcmpgtq %xmm0, %xmm2 +; SSE42-NEXT: movdqa %xmm2, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: lt_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: lt_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpcomltuq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ult <2 x i64> %a, %b + %2 = sext <2 x i1> %1 to <2 x i64> + ret <2 x i64> %2 +} + +define <4 x i32> @lt_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { +; SSE-LABEL: lt_v4i32: +; SSE: # BB#0: +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE-NEXT: pxor %xmm2, %xmm0 +; SSE-NEXT: pxor %xmm1, %xmm2 +; SSE-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: lt_v4i32: +; AVX1: # BB#0: +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vpcmpgtd %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: lt_v4i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %xmm2 +; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX2-NEXT: vpcmpgtd %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: retq +; +; XOP-LABEL: lt_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpcomltud %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: lt_v4i32: +; AVX512: # BB#0: +; AVX512-NEXT: vpbroadcastd {{.*}}(%rip), %xmm2 +; AVX512-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX512-NEXT: vpcmpgtd %xmm0, %xmm1, %xmm0 +; AVX512-NEXT: retq + %1 = icmp ult <4 x i32> %a, %b + %2 = sext <4 x i1> %1 to <4 x i32> + ret <4 x i32> %2 +} + +define <8 x i16> @lt_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { +; SSE-LABEL: lt_v8i16: +; SSE: # BB#0: +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE-NEXT: pxor %xmm2, %xmm0 +; SSE-NEXT: pxor %xmm1, %xmm2 +; SSE-NEXT: pcmpgtw %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: lt_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768] +; AVX-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpcmpgtw %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: lt_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpcomltuw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ult <8 x i16> %a, %b + %2 = sext <8 x i1> %1 to <8 x i16> + ret <8 x i16> %2 +} + +define <16 x i8> @lt_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { +; SSE-LABEL: lt_v16i8: +; SSE: # BB#0: +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128] +; SSE-NEXT: pxor %xmm2, %xmm0 +; SSE-NEXT: pxor %xmm1, %xmm2 +; SSE-NEXT: pcmpgtb %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: lt_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128] +; AVX-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpcmpgtb %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: lt_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpcomltub %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq + %1 = icmp ult <16 x i8> %a, %b + %2 = sext <16 x i1> %1 to <16 x i8> + ret <16 x i8> %2 +} diff --git a/test/CodeGen/X86/vec_ctbits.ll b/test/CodeGen/X86/vec_ctbits.ll index 318aca1d54cb..66114bc9c6bc 100644 --- a/test/CodeGen/X86/vec_ctbits.ll +++ b/test/CodeGen/X86/vec_ctbits.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1) @@ -5,25 +6,63 @@ declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>, i1) declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>) define <2 x i64> @footz(<2 x i64> %a) nounwind { +; CHECK-LABEL: footz: +; CHECK: # BB#0: +; CHECK-NEXT: movd %xmm0, %rax +; CHECK-NEXT: bsfq %rax, %rax +; CHECK-NEXT: movd %rax, %xmm1 +; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; CHECK-NEXT: movd %xmm0, %rax +; CHECK-NEXT: bsfq %rax, %rax +; CHECK-NEXT: movd %rax, %xmm0 +; CHECK-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; CHECK-NEXT: movdqa %xmm1, %xmm0 +; CHECK-NEXT: retq %c = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true) ret <2 x i64> %c -; CHECK-LABEL: footz -; CHECK: bsfq -; CHECK: bsfq } define <2 x i64> @foolz(<2 x i64> %a) nounwind { +; CHECK-LABEL: foolz: +; CHECK: # BB#0: +; CHECK-NEXT: movd %xmm0, %rax +; CHECK-NEXT: bsrq %rax, %rax +; CHECK-NEXT: xorq $63, %rax +; CHECK-NEXT: movd %rax, %xmm1 +; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; CHECK-NEXT: movd %xmm0, %rax +; CHECK-NEXT: bsrq %rax, %rax +; CHECK-NEXT: xorq $63, %rax +; CHECK-NEXT: movd %rax, %xmm0 +; CHECK-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; CHECK-NEXT: movdqa %xmm1, %xmm0 +; CHECK-NEXT: retq %c = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %a, i1 true) ret <2 x i64> %c -; CHECK-LABEL: foolz -; CHECK: bsrq -; CHECK: xorq $63 -; CHECK: bsrq -; CHECK: xorq $63 } define <2 x i64> @foopop(<2 x i64> %a) nounwind { +; CHECK-LABEL: foopop: +; CHECK: # BB#0: +; CHECK-NEXT: movdqa %xmm0, %xmm1 +; CHECK-NEXT: psrlq $1, %xmm1 +; CHECK-NEXT: pand {{.*}}(%rip), %xmm1 +; CHECK-NEXT: psubq %xmm1, %xmm0 +; CHECK-NEXT: movdqa {{.*#+}} xmm1 = [3689348814741910323,3689348814741910323] +; CHECK-NEXT: movdqa %xmm0, %xmm2 +; CHECK-NEXT: pand %xmm1, %xmm2 +; CHECK-NEXT: psrlq $2, %xmm0 +; CHECK-NEXT: pand %xmm1, %xmm0 +; CHECK-NEXT: paddq %xmm2, %xmm0 +; CHECK-NEXT: movdqa %xmm0, %xmm1 +; CHECK-NEXT: psrlq $4, %xmm1 +; CHECK-NEXT: paddq %xmm0, %xmm1 +; CHECK-NEXT: pand {{.*}}(%rip), %xmm1 +; CHECK-NEXT: pxor %xmm0, %xmm0 +; CHECK-NEXT: psadbw %xmm0, %xmm1 +; CHECK-NEXT: movdqa %xmm1, %xmm0 +; CHECK-NEXT: retq %c = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a) ret <2 x i64> %c } @@ -33,35 +72,73 @@ declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1) declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>) define <2 x i32> @promtz(<2 x i32> %a) nounwind { +; CHECK-LABEL: promtz: +; CHECK: # BB#0: +; CHECK-NEXT: por {{.*}}(%rip), %xmm0 +; CHECK-NEXT: movd %xmm0, %rax +; CHECK-NEXT: bsfq %rax, %rax +; CHECK-NEXT: movl $64, %ecx +; CHECK-NEXT: cmoveq %rcx, %rax +; CHECK-NEXT: movd %rax, %xmm1 +; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; CHECK-NEXT: movd %xmm0, %rax +; CHECK-NEXT: bsfq %rax, %rax +; CHECK-NEXT: cmoveq %rcx, %rax +; CHECK-NEXT: movd %rax, %xmm0 +; CHECK-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; CHECK-NEXT: movdqa %xmm1, %xmm0 +; CHECK-NEXT: retq %c = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false) ret <2 x i32> %c -; CHECK: .quad 4294967296 -; CHECK: .quad 4294967296 -; CHECK-LABEL: promtz -; CHECK: bsfq -; CHECK: cmov -; CHECK: bsfq -; CHECK: cmov } define <2 x i32> @promlz(<2 x i32> %a) nounwind { +; CHECK-LABEL: promlz: +; CHECK: # BB#0: +; CHECK-NEXT: pand {{.*}}(%rip), %xmm0 +; CHECK-NEXT: movd %xmm0, %rax +; CHECK-NEXT: bsrq %rax, %rax +; CHECK-NEXT: movl $127, %ecx +; CHECK-NEXT: cmoveq %rcx, %rax +; CHECK-NEXT: xorq $63, %rax +; CHECK-NEXT: movd %rax, %xmm1 +; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; CHECK-NEXT: movd %xmm0, %rax +; CHECK-NEXT: bsrq %rax, %rax +; CHECK-NEXT: cmoveq %rcx, %rax +; CHECK-NEXT: xorq $63, %rax +; CHECK-NEXT: movd %rax, %xmm0 +; CHECK-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; CHECK-NEXT: psubq {{.*}}(%rip), %xmm1 +; CHECK-NEXT: movdqa %xmm1, %xmm0 +; CHECK-NEXT: retq %c = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %a, i1 false) ret <2 x i32> %c -; CHECK: .quad 4294967295 -; CHECK: .quad 4294967295 -; CHECK: .quad 32 -; CHECK: .quad 32 -; CHECK-LABEL: promlz -; CHECK: pand -; CHECK: bsrq -; CHECK: xorq $63 -; CHECK: bsrq -; CHECK: xorq $63 -; CHECK: psub } define <2 x i32> @prompop(<2 x i32> %a) nounwind { +; CHECK-LABEL: prompop: +; CHECK: # BB#0: +; CHECK-NEXT: pand {{.*}}(%rip), %xmm0 +; CHECK-NEXT: movdqa %xmm0, %xmm1 +; CHECK-NEXT: psrlq $1, %xmm1 +; CHECK-NEXT: pand {{.*}}(%rip), %xmm1 +; CHECK-NEXT: psubq %xmm1, %xmm0 +; CHECK-NEXT: movdqa {{.*#+}} xmm1 = [3689348814741910323,3689348814741910323] +; CHECK-NEXT: movdqa %xmm0, %xmm2 +; CHECK-NEXT: pand %xmm1, %xmm2 +; CHECK-NEXT: psrlq $2, %xmm0 +; CHECK-NEXT: pand %xmm1, %xmm0 +; CHECK-NEXT: paddq %xmm2, %xmm0 +; CHECK-NEXT: movdqa %xmm0, %xmm1 +; CHECK-NEXT: psrlq $4, %xmm1 +; CHECK-NEXT: paddq %xmm0, %xmm1 +; CHECK-NEXT: pand {{.*}}(%rip), %xmm1 +; CHECK-NEXT: pxor %xmm0, %xmm0 +; CHECK-NEXT: psadbw %xmm0, %xmm1 +; CHECK-NEXT: movdqa %xmm1, %xmm0 +; CHECK-NEXT: retq %c = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %a) ret <2 x i32> %c } diff --git a/test/CodeGen/X86/vec_extract-avx.ll b/test/CodeGen/X86/vec_extract-avx.ll index fbb84170dc83..abb07233d35e 100644 --- a/test/CodeGen/X86/vec_extract-avx.ll +++ b/test/CodeGen/X86/vec_extract-avx.ll @@ -1,14 +1,18 @@ -target triple = "x86_64-unknown-unknown" - -; RUN: llc < %s -march=x86-64 -mattr=+avx | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s ; When extracting multiple consecutive elements from a larger ; vector into a smaller one, do it efficiently. We should use ; an EXTRACT_SUBVECTOR node internally rather than a bunch of -; single element extractions. +; single element extractions. ; Extracting the low elements only requires using the right kind of store. define void @low_v8f32_to_v4f32(<8 x float> %v, <4 x float>* %ptr) { +; CHECK-LABEL: low_v8f32_to_v4f32: +; CHECK: # BB#0: +; CHECK-NEXT: vmovaps %xmm0, (%rdi) +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq %ext0 = extractelement <8 x float> %v, i32 0 %ext1 = extractelement <8 x float> %v, i32 1 %ext2 = extractelement <8 x float> %v, i32 2 @@ -19,15 +23,15 @@ define void @low_v8f32_to_v4f32(<8 x float> %v, <4 x float>* %ptr) { %ins3 = insertelement <4 x float> %ins2, float %ext3, i32 3 store <4 x float> %ins3, <4 x float>* %ptr, align 16 ret void - -; CHECK-LABEL: low_v8f32_to_v4f32 -; CHECK: vmovaps -; CHECK-NEXT: vzeroupper -; CHECK-NEXT: retq } -; Extracting the high elements requires just one AVX instruction. +; Extracting the high elements requires just one AVX instruction. define void @high_v8f32_to_v4f32(<8 x float> %v, <4 x float>* %ptr) { +; CHECK-LABEL: high_v8f32_to_v4f32: +; CHECK: # BB#0: +; CHECK-NEXT: vextractf128 $1, %ymm0, (%rdi) +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq %ext0 = extractelement <8 x float> %v, i32 4 %ext1 = extractelement <8 x float> %v, i32 5 %ext2 = extractelement <8 x float> %v, i32 6 @@ -38,17 +42,17 @@ define void @high_v8f32_to_v4f32(<8 x float> %v, <4 x float>* %ptr) { %ins3 = insertelement <4 x float> %ins2, float %ext3, i32 3 store <4 x float> %ins3, <4 x float>* %ptr, align 16 ret void - -; CHECK-LABEL: high_v8f32_to_v4f32 -; CHECK: vextractf128 -; CHECK-NEXT: vzeroupper -; CHECK-NEXT: retq } ; Make sure element type doesn't alter the codegen. Note that ; if we were actually using the vector in this function and ; have AVX2, we should generate vextracti128 (the int version). define void @high_v8i32_to_v4i32(<8 x i32> %v, <4 x i32>* %ptr) { +; CHECK-LABEL: high_v8i32_to_v4i32: +; CHECK: # BB#0: +; CHECK-NEXT: vextractf128 $1, %ymm0, (%rdi) +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq %ext0 = extractelement <8 x i32> %v, i32 4 %ext1 = extractelement <8 x i32> %v, i32 5 %ext2 = extractelement <8 x i32> %v, i32 6 @@ -59,24 +63,86 @@ define void @high_v8i32_to_v4i32(<8 x i32> %v, <4 x i32>* %ptr) { %ins3 = insertelement <4 x i32> %ins2, i32 %ext3, i32 3 store <4 x i32> %ins3, <4 x i32>* %ptr, align 16 ret void - -; CHECK-LABEL: high_v8i32_to_v4i32 -; CHECK: vextractf128 -; CHECK-NEXT: vzeroupper -; CHECK-NEXT: retq } ; Make sure that element size doesn't alter the codegen. define void @high_v4f64_to_v2f64(<4 x double> %v, <2 x double>* %ptr) { +; CHECK-LABEL: high_v4f64_to_v2f64: +; CHECK: # BB#0: +; CHECK-NEXT: vextractf128 $1, %ymm0, (%rdi) +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq %ext0 = extractelement <4 x double> %v, i32 2 %ext1 = extractelement <4 x double> %v, i32 3 %ins0 = insertelement <2 x double> undef, double %ext0, i32 0 %ins1 = insertelement <2 x double> %ins0, double %ext1, i32 1 store <2 x double> %ins1, <2 x double>* %ptr, align 16 ret void +} + +; PR25320 Make sure that a widened (possibly legalized) vector correctly zero-extends upper elements. +; FIXME - Ideally these should just call VMOVD/VMOVQ/VMOVSS/VMOVSD + +define void @legal_vzmovl_2i32_8i32(<2 x i32>* %in, <8 x i32>* %out) { +; CHECK-LABEL: legal_vzmovl_2i32_8i32: +; CHECK: # BB#0: +; CHECK-NEXT: vpmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero +; CHECK-NEXT: vxorps %ymm1, %ymm1, %ymm1 +; CHECK-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6,7] +; CHECK-NEXT: vmovaps %ymm0, (%rsi) +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq + %ld = load <2 x i32>, <2 x i32>* %in, align 8 + %ext = extractelement <2 x i32> %ld, i64 0 + %ins = insertelement <8 x i32> , i32 %ext, i64 0 + store <8 x i32> %ins, <8 x i32>* %out, align 32 + ret void +} + +define void @legal_vzmovl_2i64_4i64(<2 x i64>* %in, <4 x i64>* %out) { +; CHECK-LABEL: legal_vzmovl_2i64_4i64: +; CHECK: # BB#0: +; CHECK-NEXT: vmovupd (%rdi), %xmm0 +; CHECK-NEXT: vxorpd %ymm1, %ymm1, %ymm1 +; CHECK-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3] +; CHECK-NEXT: vmovapd %ymm0, (%rsi) +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq + %ld = load <2 x i64>, <2 x i64>* %in, align 8 + %ext = extractelement <2 x i64> %ld, i64 0 + %ins = insertelement <4 x i64> , i64 %ext, i64 0 + store <4 x i64> %ins, <4 x i64>* %out, align 32 + ret void +} -; CHECK-LABEL: high_v4f64_to_v2f64 -; CHECK: vextractf128 -; CHECK-NEXT: vzeroupper -; CHECK-NEXT: retq +define void @legal_vzmovl_2f32_8f32(<2 x float>* %in, <8 x float>* %out) { +; CHECK-LABEL: legal_vzmovl_2f32_8f32: +; CHECK: # BB#0: +; CHECK-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; CHECK-NEXT: vxorps %ymm1, %ymm1, %ymm1 +; CHECK-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6,7] +; CHECK-NEXT: vmovaps %ymm0, (%rsi) +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq + %ld = load <2 x float>, <2 x float>* %in, align 8 + %ext = extractelement <2 x float> %ld, i64 0 + %ins = insertelement <8 x float> , float %ext, i64 0 + store <8 x float> %ins, <8 x float>* %out, align 32 + ret void +} + +define void @legal_vzmovl_2f64_4f64(<2 x double>* %in, <4 x double>* %out) { +; CHECK-LABEL: legal_vzmovl_2f64_4f64: +; CHECK: # BB#0: +; CHECK-NEXT: vmovupd (%rdi), %xmm0 +; CHECK-NEXT: vxorpd %ymm1, %ymm1, %ymm1 +; CHECK-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3] +; CHECK-NEXT: vmovapd %ymm0, (%rsi) +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq + %ld = load <2 x double>, <2 x double>* %in, align 8 + %ext = extractelement <2 x double> %ld, i64 0 + %ins = insertelement <4 x double> , double %ext, i64 0 + store <4 x double> %ins, <4 x double>* %out, align 32 + ret void } diff --git a/test/CodeGen/X86/vec_fabs.ll b/test/CodeGen/X86/vec_fabs.ll index 960b5f27cf53..54f33b2bd224 100644 --- a/test/CodeGen/X86/vec_fabs.ll +++ b/test/CodeGen/X86/vec_fabs.ll @@ -1,5 +1,5 @@ ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx | FileCheck %s - +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f | FileCheck %s define <2 x double> @fabs_v2f64(<2 x double> %p) { diff --git a/test/CodeGen/X86/vec_fp_to_int.ll b/test/CodeGen/X86/vec_fp_to_int.ll index 3b1b2f5c1c77..7834b2804247 100644 --- a/test/CodeGen/X86/vec_fp_to_int.ll +++ b/test/CodeGen/X86/vec_fp_to_int.ll @@ -10,19 +10,19 @@ ; Double to Signed Integer ; -define <2 x i64> @fptosi_2vf64(<2 x double> %a) { -; SSE2-LABEL: fptosi_2vf64: -; SSE2: # BB#0: -; SSE2-NEXT: cvttsd2si %xmm0, %rax -; SSE2-NEXT: movd %rax, %xmm1 -; SSE2-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] -; SSE2-NEXT: cvttsd2si %xmm0, %rax -; SSE2-NEXT: movd %rax, %xmm0 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] -; SSE2-NEXT: movdqa %xmm1, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: fptosi_2vf64: +define <2 x i64> @fptosi_2f64_to_2i64(<2 x double> %a) { +; SSE-LABEL: fptosi_2f64_to_2i64: +; SSE: # BB#0: +; SSE-NEXT: cvttsd2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm1 +; SSE-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] +; SSE-NEXT: cvttsd2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm0 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: fptosi_2f64_to_2i64: ; AVX: # BB#0: ; AVX-NEXT: vcvttsd2si %xmm0, %rax ; AVX-NEXT: vmovq %rax, %xmm1 @@ -35,19 +35,19 @@ define <2 x i64> @fptosi_2vf64(<2 x double> %a) { ret <2 x i64> %cvt } -define <4 x i32> @fptosi_2vf64_i32(<2 x double> %a) { -; SSE2-LABEL: fptosi_2vf64_i32: -; SSE2: # BB#0: -; SSE2-NEXT: cvttsd2si %xmm0, %rax -; SSE2-NEXT: movd %rax, %xmm1 -; SSE2-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] -; SSE2-NEXT: cvttsd2si %xmm0, %rax -; SSE2-NEXT: movd %rax, %xmm0 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3] -; SSE2-NEXT: retq +define <4 x i32> @fptosi_2f64_to_2i32(<2 x double> %a) { +; SSE-LABEL: fptosi_2f64_to_2i32: +; SSE: # BB#0: +; SSE-NEXT: cvttsd2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm1 +; SSE-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] +; SSE-NEXT: cvttsd2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm0 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3] +; SSE-NEXT: retq ; -; AVX-LABEL: fptosi_2vf64_i32: +; AVX-LABEL: fptosi_2f64_to_2i32: ; AVX: # BB#0: ; AVX-NEXT: vcvttsd2si %xmm0, %rax ; AVX-NEXT: vmovq %rax, %xmm1 @@ -62,26 +62,53 @@ define <4 x i32> @fptosi_2vf64_i32(<2 x double> %a) { ret <4 x i32> %ext } -define <4 x i64> @fptosi_4vf64(<4 x double> %a) { -; SSE2-LABEL: fptosi_4vf64: -; SSE2: # BB#0: -; SSE2-NEXT: cvttsd2si %xmm0, %rax -; SSE2-NEXT: movd %rax, %xmm2 -; SSE2-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] -; SSE2-NEXT: cvttsd2si %xmm0, %rax -; SSE2-NEXT: movd %rax, %xmm0 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0] -; SSE2-NEXT: cvttsd2si %xmm1, %rax -; SSE2-NEXT: movd %rax, %xmm3 -; SSE2-NEXT: shufpd {{.*#+}} xmm1 = xmm1[1,0] -; SSE2-NEXT: cvttsd2si %xmm1, %rax -; SSE2-NEXT: movd %rax, %xmm0 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0] -; SSE2-NEXT: movdqa %xmm2, %xmm0 -; SSE2-NEXT: movdqa %xmm3, %xmm1 -; SSE2-NEXT: retq -; -; AVX-LABEL: fptosi_4vf64: +define <4 x i32> @fptosi_4f64_to_2i32(<2 x double> %a) { +; SSE-LABEL: fptosi_4f64_to_2i32: +; SSE: # BB#0: +; SSE-NEXT: cvttsd2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm1 +; SSE-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] +; SSE-NEXT: cvttsd2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm0 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3] +; SSE-NEXT: cvttsd2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE-NEXT: retq +; +; AVX-LABEL: fptosi_4f64_to_2i32: +; AVX: # BB#0: +; AVX-NEXT: vcvttpd2dqy %ymm0, %xmm0 +; AVX-NEXT: vzeroupper +; AVX-NEXT: retq + %ext = shufflevector <2 x double> %a, <2 x double> undef, <4 x i32> + %cvt = fptosi <4 x double> %ext to <4 x i32> + ret <4 x i32> %cvt +} + +define <4 x i64> @fptosi_4f64_to_4i64(<4 x double> %a) { +; SSE-LABEL: fptosi_4f64_to_4i64: +; SSE: # BB#0: +; SSE-NEXT: cvttsd2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm2 +; SSE-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] +; SSE-NEXT: cvttsd2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm0 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0] +; SSE-NEXT: cvttsd2si %xmm1, %rax +; SSE-NEXT: movd %rax, %xmm3 +; SSE-NEXT: shufpd {{.*#+}} xmm1 = xmm1[1,0] +; SSE-NEXT: cvttsd2si %xmm1, %rax +; SSE-NEXT: movd %rax, %xmm0 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0] +; SSE-NEXT: movdqa %xmm2, %xmm0 +; SSE-NEXT: movdqa %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX-LABEL: fptosi_4f64_to_4i64: ; AVX: # BB#0: ; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 ; AVX-NEXT: vcvttsd2si %xmm1, %rax @@ -102,27 +129,27 @@ define <4 x i64> @fptosi_4vf64(<4 x double> %a) { ret <4 x i64> %cvt } -define <4 x i32> @fptosi_4vf64_i32(<4 x double> %a) { -; SSE2-LABEL: fptosi_4vf64_i32: -; SSE2: # BB#0: -; SSE2-NEXT: cvttsd2si %xmm1, %rax -; SSE2-NEXT: movd %rax, %xmm2 -; SSE2-NEXT: shufpd {{.*#+}} xmm1 = xmm1[1,0] -; SSE2-NEXT: cvttsd2si %xmm1, %rax -; SSE2-NEXT: movd %rax, %xmm1 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0] -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3] -; SSE2-NEXT: cvttsd2si %xmm0, %rax -; SSE2-NEXT: movd %rax, %xmm2 -; SSE2-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] -; SSE2-NEXT: cvttsd2si %xmm0, %rax -; SSE2-NEXT: movd %rax, %xmm0 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3] -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] -; SSE2-NEXT: retq -; -; AVX-LABEL: fptosi_4vf64_i32: +define <4 x i32> @fptosi_4f64_to_4i32(<4 x double> %a) { +; SSE-LABEL: fptosi_4f64_to_4i32: +; SSE: # BB#0: +; SSE-NEXT: cvttsd2si %xmm1, %rax +; SSE-NEXT: movd %rax, %xmm2 +; SSE-NEXT: shufpd {{.*#+}} xmm1 = xmm1[1,0] +; SSE-NEXT: cvttsd2si %xmm1, %rax +; SSE-NEXT: movd %rax, %xmm1 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3] +; SSE-NEXT: cvttsd2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm2 +; SSE-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] +; SSE-NEXT: cvttsd2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm0 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3] +; SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE-NEXT: retq +; +; AVX-LABEL: fptosi_4f64_to_4i32: ; AVX: # BB#0: ; AVX-NEXT: vcvttpd2dqy %ymm0, %xmm0 ; AVX-NEXT: vzeroupper @@ -135,33 +162,33 @@ define <4 x i32> @fptosi_4vf64_i32(<4 x double> %a) { ; Double to Unsigned Integer ; -define <2 x i64> @fptoui_2vf64(<2 x double> %a) { -; SSE2-LABEL: fptoui_2vf64: -; SSE2: # BB#0: -; SSE2-NEXT: movsd {{.*#+}} xmm2 = mem[0],zero -; SSE2-NEXT: movapd %xmm0, %xmm1 -; SSE2-NEXT: subsd %xmm2, %xmm1 -; SSE2-NEXT: cvttsd2si %xmm1, %rax -; SSE2-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000 -; SSE2-NEXT: xorq %rcx, %rax -; SSE2-NEXT: cvttsd2si %xmm0, %rdx -; SSE2-NEXT: ucomisd %xmm2, %xmm0 -; SSE2-NEXT: cmovaeq %rax, %rdx -; SSE2-NEXT: movd %rdx, %xmm1 -; SSE2-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] -; SSE2-NEXT: movapd %xmm0, %xmm3 -; SSE2-NEXT: subsd %xmm2, %xmm3 -; SSE2-NEXT: cvttsd2si %xmm3, %rax -; SSE2-NEXT: xorq %rcx, %rax -; SSE2-NEXT: cvttsd2si %xmm0, %rcx -; SSE2-NEXT: ucomisd %xmm2, %xmm0 -; SSE2-NEXT: cmovaeq %rax, %rcx -; SSE2-NEXT: movd %rcx, %xmm0 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] -; SSE2-NEXT: movdqa %xmm1, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: fptoui_2vf64: +define <2 x i64> @fptoui_2f64_to_2i64(<2 x double> %a) { +; SSE-LABEL: fptoui_2f64_to_2i64: +; SSE: # BB#0: +; SSE-NEXT: movsd {{.*#+}} xmm2 = mem[0],zero +; SSE-NEXT: movapd %xmm0, %xmm1 +; SSE-NEXT: subsd %xmm2, %xmm1 +; SSE-NEXT: cvttsd2si %xmm1, %rax +; SSE-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000 +; SSE-NEXT: xorq %rcx, %rax +; SSE-NEXT: cvttsd2si %xmm0, %rdx +; SSE-NEXT: ucomisd %xmm2, %xmm0 +; SSE-NEXT: cmovaeq %rax, %rdx +; SSE-NEXT: movd %rdx, %xmm1 +; SSE-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] +; SSE-NEXT: movapd %xmm0, %xmm3 +; SSE-NEXT: subsd %xmm2, %xmm3 +; SSE-NEXT: cvttsd2si %xmm3, %rax +; SSE-NEXT: xorq %rcx, %rax +; SSE-NEXT: cvttsd2si %xmm0, %rcx +; SSE-NEXT: ucomisd %xmm2, %xmm0 +; SSE-NEXT: cmovaeq %rax, %rcx +; SSE-NEXT: movd %rcx, %xmm0 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: fptoui_2f64_to_2i64: ; AVX: # BB#0: ; AVX-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero ; AVX-NEXT: vsubsd %xmm1, %xmm0, %xmm2 @@ -186,33 +213,33 @@ define <2 x i64> @fptoui_2vf64(<2 x double> %a) { ret <2 x i64> %cvt } -define <4 x i32> @fptoui_2vf64_i32(<2 x double> %a) { -; SSE2-LABEL: fptoui_2vf64_i32: -; SSE2: # BB#0: -; SSE2-NEXT: movsd {{.*#+}} xmm1 = mem[0],zero -; SSE2-NEXT: movapd %xmm0, %xmm2 -; SSE2-NEXT: subsd %xmm1, %xmm2 -; SSE2-NEXT: cvttsd2si %xmm2, %rax -; SSE2-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000 -; SSE2-NEXT: xorq %rcx, %rax -; SSE2-NEXT: cvttsd2si %xmm0, %rdx -; SSE2-NEXT: ucomisd %xmm1, %xmm0 -; SSE2-NEXT: cmovaeq %rax, %rdx -; SSE2-NEXT: movd %rdx, %xmm2 -; SSE2-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] -; SSE2-NEXT: movapd %xmm0, %xmm3 -; SSE2-NEXT: subsd %xmm1, %xmm3 -; SSE2-NEXT: cvttsd2si %xmm3, %rax -; SSE2-NEXT: xorq %rcx, %rax -; SSE2-NEXT: cvttsd2si %xmm0, %rcx -; SSE2-NEXT: ucomisd %xmm1, %xmm0 -; SSE2-NEXT: cmovaeq %rax, %rcx -; SSE2-NEXT: movd %rcx, %xmm0 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3] -; SSE2-NEXT: retq -; -; AVX-LABEL: fptoui_2vf64_i32: +define <4 x i32> @fptoui_2f64_to_2i32(<2 x double> %a) { +; SSE-LABEL: fptoui_2f64_to_2i32: +; SSE: # BB#0: +; SSE-NEXT: movsd {{.*#+}} xmm1 = mem[0],zero +; SSE-NEXT: movapd %xmm0, %xmm2 +; SSE-NEXT: subsd %xmm1, %xmm2 +; SSE-NEXT: cvttsd2si %xmm2, %rax +; SSE-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000 +; SSE-NEXT: xorq %rcx, %rax +; SSE-NEXT: cvttsd2si %xmm0, %rdx +; SSE-NEXT: ucomisd %xmm1, %xmm0 +; SSE-NEXT: cmovaeq %rax, %rdx +; SSE-NEXT: movd %rdx, %xmm2 +; SSE-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] +; SSE-NEXT: movapd %xmm0, %xmm3 +; SSE-NEXT: subsd %xmm1, %xmm3 +; SSE-NEXT: cvttsd2si %xmm3, %rax +; SSE-NEXT: xorq %rcx, %rax +; SSE-NEXT: cvttsd2si %xmm0, %rcx +; SSE-NEXT: ucomisd %xmm1, %xmm0 +; SSE-NEXT: cmovaeq %rax, %rcx +; SSE-NEXT: movd %rcx, %xmm0 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3] +; SSE-NEXT: retq +; +; AVX-LABEL: fptoui_2f64_to_2i32: ; AVX: # BB#0: ; AVX-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero ; AVX-NEXT: vsubsd %xmm1, %xmm0, %xmm2 @@ -239,51 +266,101 @@ define <4 x i32> @fptoui_2vf64_i32(<2 x double> %a) { ret <4 x i32> %ext } -define <4 x i64> @fptoui_4vf64(<4 x double> %a) { -; SSE2-LABEL: fptoui_4vf64: -; SSE2: # BB#0: -; SSE2-NEXT: movapd %xmm0, %xmm2 -; SSE2-NEXT: movsd {{.*#+}} xmm3 = mem[0],zero -; SSE2-NEXT: subsd %xmm3, %xmm0 -; SSE2-NEXT: cvttsd2si %xmm0, %rcx -; SSE2-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000 -; SSE2-NEXT: xorq %rax, %rcx -; SSE2-NEXT: cvttsd2si %xmm2, %rdx -; SSE2-NEXT: ucomisd %xmm3, %xmm2 -; SSE2-NEXT: cmovaeq %rcx, %rdx -; SSE2-NEXT: movd %rdx, %xmm0 -; SSE2-NEXT: shufpd {{.*#+}} xmm2 = xmm2[1,0] -; SSE2-NEXT: movapd %xmm2, %xmm4 -; SSE2-NEXT: subsd %xmm3, %xmm4 -; SSE2-NEXT: cvttsd2si %xmm4, %rcx -; SSE2-NEXT: xorq %rax, %rcx -; SSE2-NEXT: cvttsd2si %xmm2, %rdx -; SSE2-NEXT: ucomisd %xmm3, %xmm2 -; SSE2-NEXT: cmovaeq %rcx, %rdx -; SSE2-NEXT: movd %rdx, %xmm2 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] -; SSE2-NEXT: movapd %xmm1, %xmm2 -; SSE2-NEXT: subsd %xmm3, %xmm2 -; SSE2-NEXT: cvttsd2si %xmm2, %rcx -; SSE2-NEXT: xorq %rax, %rcx -; SSE2-NEXT: cvttsd2si %xmm1, %rdx -; SSE2-NEXT: ucomisd %xmm3, %xmm1 -; SSE2-NEXT: cmovaeq %rcx, %rdx -; SSE2-NEXT: movd %rdx, %xmm2 -; SSE2-NEXT: shufpd {{.*#+}} xmm1 = xmm1[1,0] -; SSE2-NEXT: movapd %xmm1, %xmm4 -; SSE2-NEXT: subsd %xmm3, %xmm4 -; SSE2-NEXT: cvttsd2si %xmm4, %rcx -; SSE2-NEXT: xorq %rax, %rcx -; SSE2-NEXT: cvttsd2si %xmm1, %rax -; SSE2-NEXT: ucomisd %xmm3, %xmm1 -; SSE2-NEXT: cmovaeq %rcx, %rax -; SSE2-NEXT: movd %rax, %xmm1 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0] -; SSE2-NEXT: movdqa %xmm2, %xmm1 -; SSE2-NEXT: retq -; -; AVX-LABEL: fptoui_4vf64: +define <4 x i32> @fptoui_4f64_to_2i32(<2 x double> %a) { +; SSE-LABEL: fptoui_4f64_to_2i32: +; SSE: # BB#0: +; SSE-NEXT: movsd {{.*#+}} xmm1 = mem[0],zero +; SSE-NEXT: movapd %xmm0, %xmm2 +; SSE-NEXT: subsd %xmm1, %xmm2 +; SSE-NEXT: cvttsd2si %xmm2, %rax +; SSE-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000 +; SSE-NEXT: xorq %rcx, %rax +; SSE-NEXT: cvttsd2si %xmm0, %rdx +; SSE-NEXT: ucomisd %xmm1, %xmm0 +; SSE-NEXT: cmovaeq %rax, %rdx +; SSE-NEXT: movd %rdx, %xmm2 +; SSE-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] +; SSE-NEXT: movapd %xmm0, %xmm3 +; SSE-NEXT: subsd %xmm1, %xmm3 +; SSE-NEXT: cvttsd2si %xmm3, %rax +; SSE-NEXT: xorq %rcx, %rax +; SSE-NEXT: cvttsd2si %xmm0, %rdx +; SSE-NEXT: ucomisd %xmm1, %xmm0 +; SSE-NEXT: cmovaeq %rax, %rdx +; SSE-NEXT: movd %rdx, %xmm0 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3] +; SSE-NEXT: cvttsd2si %xmm0, %rax +; SSE-NEXT: xorq %rax, %rcx +; SSE-NEXT: ucomisd %xmm1, %xmm0 +; SSE-NEXT: cmovbq %rax, %rcx +; SSE-NEXT: movd %rcx, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE-NEXT: retq +; +; AVX-LABEL: fptoui_4f64_to_2i32: +; AVX: # BB#0: +; AVX-NEXT: vpermilpd {{.*#+}} xmm1 = xmm0[1,0] +; AVX-NEXT: vcvttsd2si %xmm1, %rax +; AVX-NEXT: vcvttsd2si %xmm0, %rcx +; AVX-NEXT: vmovd %ecx, %xmm0 +; AVX-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0 +; AVX-NEXT: vcvttsd2si %xmm0, %rax +; AVX-NEXT: vpinsrd $2, %eax, %xmm0, %xmm0 +; AVX-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0 +; AVX-NEXT: retq + %ext = shufflevector <2 x double> %a, <2 x double> undef, <4 x i32> + %cvt = fptoui <4 x double> %ext to <4 x i32> + ret <4 x i32> %cvt +} + +define <4 x i64> @fptoui_4f64_to_4i64(<4 x double> %a) { +; SSE-LABEL: fptoui_4f64_to_4i64: +; SSE: # BB#0: +; SSE-NEXT: movapd %xmm0, %xmm2 +; SSE-NEXT: movsd {{.*#+}} xmm3 = mem[0],zero +; SSE-NEXT: subsd %xmm3, %xmm0 +; SSE-NEXT: cvttsd2si %xmm0, %rcx +; SSE-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000 +; SSE-NEXT: xorq %rax, %rcx +; SSE-NEXT: cvttsd2si %xmm2, %rdx +; SSE-NEXT: ucomisd %xmm3, %xmm2 +; SSE-NEXT: cmovaeq %rcx, %rdx +; SSE-NEXT: movd %rdx, %xmm0 +; SSE-NEXT: shufpd {{.*#+}} xmm2 = xmm2[1,0] +; SSE-NEXT: movapd %xmm2, %xmm4 +; SSE-NEXT: subsd %xmm3, %xmm4 +; SSE-NEXT: cvttsd2si %xmm4, %rcx +; SSE-NEXT: xorq %rax, %rcx +; SSE-NEXT: cvttsd2si %xmm2, %rdx +; SSE-NEXT: ucomisd %xmm3, %xmm2 +; SSE-NEXT: cmovaeq %rcx, %rdx +; SSE-NEXT: movd %rdx, %xmm2 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] +; SSE-NEXT: movapd %xmm1, %xmm2 +; SSE-NEXT: subsd %xmm3, %xmm2 +; SSE-NEXT: cvttsd2si %xmm2, %rcx +; SSE-NEXT: xorq %rax, %rcx +; SSE-NEXT: cvttsd2si %xmm1, %rdx +; SSE-NEXT: ucomisd %xmm3, %xmm1 +; SSE-NEXT: cmovaeq %rcx, %rdx +; SSE-NEXT: movd %rdx, %xmm2 +; SSE-NEXT: shufpd {{.*#+}} xmm1 = xmm1[1,0] +; SSE-NEXT: movapd %xmm1, %xmm4 +; SSE-NEXT: subsd %xmm3, %xmm4 +; SSE-NEXT: cvttsd2si %xmm4, %rcx +; SSE-NEXT: xorq %rax, %rcx +; SSE-NEXT: cvttsd2si %xmm1, %rax +; SSE-NEXT: ucomisd %xmm3, %xmm1 +; SSE-NEXT: cmovaeq %rcx, %rax +; SSE-NEXT: movd %rax, %xmm1 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0] +; SSE-NEXT: movdqa %xmm2, %xmm1 +; SSE-NEXT: retq +; +; AVX-LABEL: fptoui_4f64_to_4i64: ; AVX: # BB#0: ; AVX-NEXT: vextractf128 $1, %ymm0, %xmm2 ; AVX-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero @@ -326,53 +403,53 @@ define <4 x i64> @fptoui_4vf64(<4 x double> %a) { ret <4 x i64> %cvt } -define <4 x i32> @fptoui_4vf64_i32(<4 x double> %a) { -; SSE2-LABEL: fptoui_4vf64_i32: -; SSE2: # BB#0: -; SSE2-NEXT: movsd {{.*#+}} xmm2 = mem[0],zero -; SSE2-NEXT: movapd %xmm1, %xmm3 -; SSE2-NEXT: subsd %xmm2, %xmm3 -; SSE2-NEXT: cvttsd2si %xmm3, %rcx -; SSE2-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000 -; SSE2-NEXT: xorq %rax, %rcx -; SSE2-NEXT: cvttsd2si %xmm1, %rdx -; SSE2-NEXT: ucomisd %xmm2, %xmm1 -; SSE2-NEXT: cmovaeq %rcx, %rdx -; SSE2-NEXT: movd %rdx, %xmm3 -; SSE2-NEXT: shufpd {{.*#+}} xmm1 = xmm1[1,0] -; SSE2-NEXT: movapd %xmm1, %xmm4 -; SSE2-NEXT: subsd %xmm2, %xmm4 -; SSE2-NEXT: cvttsd2si %xmm4, %rcx -; SSE2-NEXT: xorq %rax, %rcx -; SSE2-NEXT: cvttsd2si %xmm1, %rdx -; SSE2-NEXT: ucomisd %xmm2, %xmm1 -; SSE2-NEXT: cmovaeq %rcx, %rdx -; SSE2-NEXT: movd %rdx, %xmm1 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0] -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3] -; SSE2-NEXT: movapd %xmm0, %xmm3 -; SSE2-NEXT: subsd %xmm2, %xmm3 -; SSE2-NEXT: cvttsd2si %xmm3, %rcx -; SSE2-NEXT: xorq %rax, %rcx -; SSE2-NEXT: cvttsd2si %xmm0, %rdx -; SSE2-NEXT: ucomisd %xmm2, %xmm0 -; SSE2-NEXT: cmovaeq %rcx, %rdx -; SSE2-NEXT: movd %rdx, %xmm3 -; SSE2-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] -; SSE2-NEXT: movapd %xmm0, %xmm4 -; SSE2-NEXT: subsd %xmm2, %xmm4 -; SSE2-NEXT: cvttsd2si %xmm4, %rcx -; SSE2-NEXT: xorq %rax, %rcx -; SSE2-NEXT: cvttsd2si %xmm0, %rax -; SSE2-NEXT: ucomisd %xmm2, %xmm0 -; SSE2-NEXT: cmovaeq %rcx, %rax -; SSE2-NEXT: movd %rax, %xmm0 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3] -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] -; SSE2-NEXT: retq -; -; AVX-LABEL: fptoui_4vf64_i32: +define <4 x i32> @fptoui_4f64_to_4i32(<4 x double> %a) { +; SSE-LABEL: fptoui_4f64_to_4i32: +; SSE: # BB#0: +; SSE-NEXT: movsd {{.*#+}} xmm2 = mem[0],zero +; SSE-NEXT: movapd %xmm1, %xmm3 +; SSE-NEXT: subsd %xmm2, %xmm3 +; SSE-NEXT: cvttsd2si %xmm3, %rcx +; SSE-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000 +; SSE-NEXT: xorq %rax, %rcx +; SSE-NEXT: cvttsd2si %xmm1, %rdx +; SSE-NEXT: ucomisd %xmm2, %xmm1 +; SSE-NEXT: cmovaeq %rcx, %rdx +; SSE-NEXT: movd %rdx, %xmm3 +; SSE-NEXT: shufpd {{.*#+}} xmm1 = xmm1[1,0] +; SSE-NEXT: movapd %xmm1, %xmm4 +; SSE-NEXT: subsd %xmm2, %xmm4 +; SSE-NEXT: cvttsd2si %xmm4, %rcx +; SSE-NEXT: xorq %rax, %rcx +; SSE-NEXT: cvttsd2si %xmm1, %rdx +; SSE-NEXT: ucomisd %xmm2, %xmm1 +; SSE-NEXT: cmovaeq %rcx, %rdx +; SSE-NEXT: movd %rdx, %xmm1 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3] +; SSE-NEXT: movapd %xmm0, %xmm3 +; SSE-NEXT: subsd %xmm2, %xmm3 +; SSE-NEXT: cvttsd2si %xmm3, %rcx +; SSE-NEXT: xorq %rax, %rcx +; SSE-NEXT: cvttsd2si %xmm0, %rdx +; SSE-NEXT: ucomisd %xmm2, %xmm0 +; SSE-NEXT: cmovaeq %rcx, %rdx +; SSE-NEXT: movd %rdx, %xmm3 +; SSE-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] +; SSE-NEXT: movapd %xmm0, %xmm4 +; SSE-NEXT: subsd %xmm2, %xmm4 +; SSE-NEXT: cvttsd2si %xmm4, %rcx +; SSE-NEXT: xorq %rax, %rcx +; SSE-NEXT: cvttsd2si %xmm0, %rax +; SSE-NEXT: ucomisd %xmm2, %xmm0 +; SSE-NEXT: cmovaeq %rcx, %rax +; SSE-NEXT: movd %rax, %xmm0 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3] +; SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE-NEXT: retq +; +; AVX-LABEL: fptoui_4f64_to_4i32: ; AVX: # BB#0: ; AVX-NEXT: vpermilpd {{.*#+}} xmm1 = xmm0[1,0] ; AVX-NEXT: vcvttsd2si %xmm1, %rax @@ -395,13 +472,13 @@ define <4 x i32> @fptoui_4vf64_i32(<4 x double> %a) { ; Float to Signed Integer ; -define <4 x i32> @fptosi_4vf32(<4 x float> %a) { -; SSE2-LABEL: fptosi_4vf32: -; SSE2: # BB#0: -; SSE2-NEXT: cvttps2dq %xmm0, %xmm0 -; SSE2-NEXT: retq +define <4 x i32> @fptosi_4f32_to_4i32(<4 x float> %a) { +; SSE-LABEL: fptosi_4f32_to_4i32: +; SSE: # BB#0: +; SSE-NEXT: cvttps2dq %xmm0, %xmm0 +; SSE-NEXT: retq ; -; AVX-LABEL: fptosi_4vf32: +; AVX-LABEL: fptosi_4f32_to_4i32: ; AVX: # BB#0: ; AVX-NEXT: vcvttps2dq %xmm0, %xmm0 ; AVX-NEXT: retq @@ -409,19 +486,19 @@ define <4 x i32> @fptosi_4vf32(<4 x float> %a) { ret <4 x i32> %cvt } -define <2 x i64> @fptosi_4vf32_i64(<4 x float> %a) { -; SSE2-LABEL: fptosi_4vf32_i64: -; SSE2: # BB#0: -; SSE2-NEXT: cvttss2si %xmm0, %rax -; SSE2-NEXT: movd %rax, %xmm1 -; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,2,3] -; SSE2-NEXT: cvttss2si %xmm0, %rax -; SSE2-NEXT: movd %rax, %xmm0 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] -; SSE2-NEXT: movdqa %xmm1, %xmm0 -; SSE2-NEXT: retq +define <2 x i64> @fptosi_2f32_to_2i64(<4 x float> %a) { +; SSE-LABEL: fptosi_2f32_to_2i64: +; SSE: # BB#0: +; SSE-NEXT: cvttss2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,2,3] +; SSE-NEXT: cvttss2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm0 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: retq ; -; AVX-LABEL: fptosi_4vf32_i64: +; AVX-LABEL: fptosi_2f32_to_2i64: ; AVX: # BB#0: ; AVX-NEXT: vcvttss2si %xmm0, %rax ; AVX-NEXT: vmovq %rax, %xmm1 @@ -430,19 +507,45 @@ define <2 x i64> @fptosi_4vf32_i64(<4 x float> %a) { ; AVX-NEXT: vmovq %rax, %xmm0 ; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] ; AVX-NEXT: retq - %shuf = shufflevector <4 x float> %a, <4 x float> %a, <2 x i32> + %shuf = shufflevector <4 x float> %a, <4 x float> undef, <2 x i32> %cvt = fptosi <2 x float> %shuf to <2 x i64> ret <2 x i64> %cvt } -define <8 x i32> @fptosi_8vf32(<8 x float> %a) { -; SSE2-LABEL: fptosi_8vf32: -; SSE2: # BB#0: -; SSE2-NEXT: cvttps2dq %xmm0, %xmm0 -; SSE2-NEXT: cvttps2dq %xmm1, %xmm1 -; SSE2-NEXT: retq +define <2 x i64> @fptosi_4f32_to_2i64(<4 x float> %a) { +; SSE-LABEL: fptosi_4f32_to_2i64: +; SSE: # BB#0: +; SSE-NEXT: cvttss2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,2,3] +; SSE-NEXT: cvttss2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm0 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: fptosi_4f32_to_2i64: +; AVX: # BB#0: +; AVX-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3] +; AVX-NEXT: vcvttss2si %xmm1, %rax +; AVX-NEXT: vcvttss2si %xmm0, %rcx +; AVX-NEXT: vmovq %rcx, %xmm0 +; AVX-NEXT: vmovq %rax, %xmm1 +; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX-NEXT: retq + %cvt = fptosi <4 x float> %a to <4 x i64> + %shuf = shufflevector <4 x i64> %cvt, <4 x i64> undef, <2 x i32> + ret <2 x i64> %shuf +} + +define <8 x i32> @fptosi_8f32_to_8i32(<8 x float> %a) { +; SSE-LABEL: fptosi_8f32_to_8i32: +; SSE: # BB#0: +; SSE-NEXT: cvttps2dq %xmm0, %xmm0 +; SSE-NEXT: cvttps2dq %xmm1, %xmm1 +; SSE-NEXT: retq ; -; AVX-LABEL: fptosi_8vf32: +; AVX-LABEL: fptosi_8f32_to_8i32: ; AVX: # BB#0: ; AVX-NEXT: vcvttps2dq %ymm0, %ymm0 ; AVX-NEXT: retq @@ -450,28 +553,28 @@ define <8 x i32> @fptosi_8vf32(<8 x float> %a) { ret <8 x i32> %cvt } -define <4 x i64> @fptosi_8vf32_i64(<8 x float> %a) { -; SSE2-LABEL: fptosi_8vf32_i64: -; SSE2: # BB#0: -; SSE2-NEXT: cvttss2si %xmm0, %rax -; SSE2-NEXT: movd %rax, %xmm2 -; SSE2-NEXT: movaps %xmm0, %xmm1 -; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1,2,3] -; SSE2-NEXT: cvttss2si %xmm1, %rax -; SSE2-NEXT: movd %rax, %xmm1 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0] -; SSE2-NEXT: movaps %xmm0, %xmm1 -; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1,2,3] -; SSE2-NEXT: cvttss2si %xmm1, %rax -; SSE2-NEXT: movd %rax, %xmm3 -; SSE2-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] -; SSE2-NEXT: cvttss2si %xmm0, %rax -; SSE2-NEXT: movd %rax, %xmm1 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0] -; SSE2-NEXT: movdqa %xmm2, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: fptosi_8vf32_i64: +define <4 x i64> @fptosi_4f32_to_4i64(<8 x float> %a) { +; SSE-LABEL: fptosi_4f32_to_4i64: +; SSE: # BB#0: +; SSE-NEXT: cvttss2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm2 +; SSE-NEXT: movaps %xmm0, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1,2,3] +; SSE-NEXT: cvttss2si %xmm1, %rax +; SSE-NEXT: movd %rax, %xmm1 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0] +; SSE-NEXT: movaps %xmm0, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1,2,3] +; SSE-NEXT: cvttss2si %xmm1, %rax +; SSE-NEXT: movd %rax, %xmm3 +; SSE-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] +; SSE-NEXT: cvttss2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm1 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0] +; SSE-NEXT: movdqa %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: fptosi_4f32_to_4i64: ; AVX: # BB#0: ; AVX-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3] ; AVX-NEXT: vcvttss2si %xmm1, %rax @@ -488,38 +591,81 @@ define <4 x i64> @fptosi_8vf32_i64(<8 x float> %a) { ; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0] ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX-NEXT: retq - %shuf = shufflevector <8 x float> %a, <8 x float> %a, <4 x i32> + %shuf = shufflevector <8 x float> %a, <8 x float> undef, <4 x i32> %cvt = fptosi <4 x float> %shuf to <4 x i64> ret <4 x i64> %cvt } +define <4 x i64> @fptosi_8f32_to_4i64(<8 x float> %a) { +; SSE-LABEL: fptosi_8f32_to_4i64: +; SSE: # BB#0: +; SSE-NEXT: cvttss2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm2 +; SSE-NEXT: movaps %xmm0, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1,2,3] +; SSE-NEXT: cvttss2si %xmm1, %rax +; SSE-NEXT: movd %rax, %xmm1 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0] +; SSE-NEXT: movaps %xmm0, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1,2,3] +; SSE-NEXT: cvttss2si %xmm1, %rax +; SSE-NEXT: movd %rax, %xmm3 +; SSE-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] +; SSE-NEXT: cvttss2si %xmm0, %rax +; SSE-NEXT: movd %rax, %xmm1 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0] +; SSE-NEXT: movdqa %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: fptosi_8f32_to_4i64: +; AVX: # BB#0: +; AVX-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3] +; AVX-NEXT: vcvttss2si %xmm1, %rax +; AVX-NEXT: vmovq %rax, %xmm1 +; AVX-NEXT: vpermilpd {{.*#+}} xmm2 = xmm0[1,0] +; AVX-NEXT: vcvttss2si %xmm2, %rax +; AVX-NEXT: vmovq %rax, %xmm2 +; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX-NEXT: vcvttss2si %xmm0, %rax +; AVX-NEXT: vmovq %rax, %xmm2 +; AVX-NEXT: vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3] +; AVX-NEXT: vcvttss2si %xmm0, %rax +; AVX-NEXT: vmovq %rax, %xmm0 +; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0] +; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX-NEXT: retq + %cvt = fptosi <8 x float> %a to <8 x i64> + %shuf = shufflevector <8 x i64> %cvt, <8 x i64> undef, <4 x i32> + ret <4 x i64> %shuf +} + ; ; Float to Unsigned Integer ; -define <4 x i32> @fptoui_4vf32(<4 x float> %a) { -; SSE2-LABEL: fptoui_4vf32: -; SSE2: # BB#0: -; SSE2-NEXT: movaps %xmm0, %xmm1 -; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1,2,3] -; SSE2-NEXT: cvttss2si %xmm1, %rax -; SSE2-NEXT: movd %eax, %xmm1 -; SSE2-NEXT: movaps %xmm0, %xmm2 -; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1,2,3] -; SSE2-NEXT: cvttss2si %xmm2, %rax -; SSE2-NEXT: movd %eax, %xmm2 -; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; SSE2-NEXT: cvttss2si %xmm0, %rax -; SSE2-NEXT: movd %eax, %xmm1 -; SSE2-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] -; SSE2-NEXT: cvttss2si %xmm0, %rax -; SSE2-NEXT: movd %eax, %xmm0 -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] -; SSE2-NEXT: movdqa %xmm1, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: fptoui_4vf32: +define <4 x i32> @fptoui_4f32_to_4i32(<4 x float> %a) { +; SSE-LABEL: fptoui_4f32_to_4i32: +; SSE: # BB#0: +; SSE-NEXT: movaps %xmm0, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,1,2,3] +; SSE-NEXT: cvttss2si %xmm1, %rax +; SSE-NEXT: movd %eax, %xmm1 +; SSE-NEXT: movaps %xmm0, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1,2,3] +; SSE-NEXT: cvttss2si %xmm2, %rax +; SSE-NEXT: movd %eax, %xmm2 +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE-NEXT: cvttss2si %xmm0, %rax +; SSE-NEXT: movd %eax, %xmm1 +; SSE-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] +; SSE-NEXT: cvttss2si %xmm0, %rax +; SSE-NEXT: movd %eax, %xmm0 +; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: fptoui_4f32_to_4i32: ; AVX: # BB#0: ; AVX-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3] ; AVX-NEXT: vcvttss2si %xmm1, %rax @@ -537,33 +683,33 @@ define <4 x i32> @fptoui_4vf32(<4 x float> %a) { ret <4 x i32> %cvt } -define <2 x i64> @fptoui_4vf32_i64(<4 x float> %a) { -; SSE2-LABEL: fptoui_4vf32_i64: -; SSE2: # BB#0: -; SSE2-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero -; SSE2-NEXT: movaps %xmm0, %xmm1 -; SSE2-NEXT: subss %xmm2, %xmm1 -; SSE2-NEXT: cvttss2si %xmm1, %rax -; SSE2-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000 -; SSE2-NEXT: xorq %rcx, %rax -; SSE2-NEXT: cvttss2si %xmm0, %rdx -; SSE2-NEXT: ucomiss %xmm2, %xmm0 -; SSE2-NEXT: cmovaeq %rax, %rdx -; SSE2-NEXT: movd %rdx, %xmm1 -; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,2,3] -; SSE2-NEXT: movaps %xmm0, %xmm3 -; SSE2-NEXT: subss %xmm2, %xmm3 -; SSE2-NEXT: cvttss2si %xmm3, %rax -; SSE2-NEXT: xorq %rcx, %rax -; SSE2-NEXT: cvttss2si %xmm0, %rcx -; SSE2-NEXT: ucomiss %xmm2, %xmm0 -; SSE2-NEXT: cmovaeq %rax, %rcx -; SSE2-NEXT: movd %rcx, %xmm0 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] -; SSE2-NEXT: movdqa %xmm1, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: fptoui_4vf32_i64: +define <2 x i64> @fptoui_2f32_to_2i64(<4 x float> %a) { +; SSE-LABEL: fptoui_2f32_to_2i64: +; SSE: # BB#0: +; SSE-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero +; SSE-NEXT: movaps %xmm0, %xmm1 +; SSE-NEXT: subss %xmm2, %xmm1 +; SSE-NEXT: cvttss2si %xmm1, %rax +; SSE-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000 +; SSE-NEXT: xorq %rcx, %rax +; SSE-NEXT: cvttss2si %xmm0, %rdx +; SSE-NEXT: ucomiss %xmm2, %xmm0 +; SSE-NEXT: cmovaeq %rax, %rdx +; SSE-NEXT: movd %rdx, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,2,3] +; SSE-NEXT: movaps %xmm0, %xmm3 +; SSE-NEXT: subss %xmm2, %xmm3 +; SSE-NEXT: cvttss2si %xmm3, %rax +; SSE-NEXT: xorq %rcx, %rax +; SSE-NEXT: cvttss2si %xmm0, %rcx +; SSE-NEXT: ucomiss %xmm2, %xmm0 +; SSE-NEXT: cmovaeq %rax, %rcx +; SSE-NEXT: movd %rcx, %xmm0 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: fptoui_2f32_to_2i64: ; AVX: # BB#0: ; AVX-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero ; AVX-NEXT: vsubss %xmm1, %xmm0, %xmm2 @@ -584,50 +730,102 @@ define <2 x i64> @fptoui_4vf32_i64(<4 x float> %a) { ; AVX-NEXT: vmovq %rcx, %xmm0 ; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0] ; AVX-NEXT: retq - %shuf = shufflevector <4 x float> %a, <4 x float> %a, <2 x i32> + %shuf = shufflevector <4 x float> %a, <4 x float> undef, <2 x i32> %cvt = fptoui <2 x float> %shuf to <2 x i64> ret <2 x i64> %cvt } -define <8 x i32> @fptoui_8vf32(<8 x float> %a) { -; SSE2-LABEL: fptoui_8vf32: -; SSE2: # BB#0: -; SSE2-NEXT: movaps %xmm0, %xmm2 -; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,1,2,3] -; SSE2-NEXT: cvttss2si %xmm0, %rax -; SSE2-NEXT: movd %eax, %xmm0 -; SSE2-NEXT: movaps %xmm2, %xmm3 -; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,1,2,3] -; SSE2-NEXT: cvttss2si %xmm3, %rax -; SSE2-NEXT: movd %eax, %xmm3 -; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] -; SSE2-NEXT: cvttss2si %xmm2, %rax -; SSE2-NEXT: movd %eax, %xmm0 -; SSE2-NEXT: shufpd {{.*#+}} xmm2 = xmm2[1,0] -; SSE2-NEXT: cvttss2si %xmm2, %rax -; SSE2-NEXT: movd %eax, %xmm2 -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1] -; SSE2-NEXT: movaps %xmm1, %xmm2 -; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,1,2,3] -; SSE2-NEXT: cvttss2si %xmm2, %rax -; SSE2-NEXT: movd %eax, %xmm2 -; SSE2-NEXT: movaps %xmm1, %xmm3 -; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,1,2,3] -; SSE2-NEXT: cvttss2si %xmm3, %rax -; SSE2-NEXT: movd %eax, %xmm3 -; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; SSE2-NEXT: cvttss2si %xmm1, %rax -; SSE2-NEXT: movd %eax, %xmm2 -; SSE2-NEXT: shufpd {{.*#+}} xmm1 = xmm1[1,0] -; SSE2-NEXT: cvttss2si %xmm1, %rax -; SSE2-NEXT: movd %eax, %xmm1 -; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] -; SSE2-NEXT: movdqa %xmm2, %xmm1 -; SSE2-NEXT: retq -; -; AVX-LABEL: fptoui_8vf32: +define <2 x i64> @fptoui_4f32_to_2i64(<4 x float> %a) { +; SSE-LABEL: fptoui_4f32_to_2i64: +; SSE: # BB#0: +; SSE-NEXT: movss {{.*#+}} xmm2 = mem[0],zero,zero,zero +; SSE-NEXT: movaps %xmm0, %xmm1 +; SSE-NEXT: subss %xmm2, %xmm1 +; SSE-NEXT: cvttss2si %xmm1, %rax +; SSE-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000 +; SSE-NEXT: xorq %rcx, %rax +; SSE-NEXT: cvttss2si %xmm0, %rdx +; SSE-NEXT: ucomiss %xmm2, %xmm0 +; SSE-NEXT: cmovaeq %rax, %rdx +; SSE-NEXT: movd %rdx, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,2,3] +; SSE-NEXT: movaps %xmm0, %xmm3 +; SSE-NEXT: subss %xmm2, %xmm3 +; SSE-NEXT: cvttss2si %xmm3, %rax +; SSE-NEXT: xorq %rcx, %rax +; SSE-NEXT: cvttss2si %xmm0, %rcx +; SSE-NEXT: ucomiss %xmm2, %xmm0 +; SSE-NEXT: cmovaeq %rax, %rcx +; SSE-NEXT: movd %rcx, %xmm0 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: fptoui_4f32_to_2i64: +; AVX: # BB#0: +; AVX-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3] +; AVX-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero +; AVX-NEXT: vsubss %xmm2, %xmm1, %xmm3 +; AVX-NEXT: vcvttss2si %xmm3, %rax +; AVX-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000 +; AVX-NEXT: xorq %rcx, %rax +; AVX-NEXT: vcvttss2si %xmm1, %rdx +; AVX-NEXT: vucomiss %xmm2, %xmm1 +; AVX-NEXT: cmovaeq %rax, %rdx +; AVX-NEXT: vsubss %xmm2, %xmm0, %xmm1 +; AVX-NEXT: vcvttss2si %xmm1, %rax +; AVX-NEXT: xorq %rcx, %rax +; AVX-NEXT: vcvttss2si %xmm0, %rcx +; AVX-NEXT: vucomiss %xmm2, %xmm0 +; AVX-NEXT: cmovaeq %rax, %rcx +; AVX-NEXT: vmovq %rcx, %xmm0 +; AVX-NEXT: vmovq %rdx, %xmm1 +; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX-NEXT: retq + %cvt = fptoui <4 x float> %a to <4 x i64> + %shuf = shufflevector <4 x i64> %cvt, <4 x i64> undef, <2 x i32> + ret <2 x i64> %shuf +} + +define <8 x i32> @fptoui_8f32_to_8i32(<8 x float> %a) { +; SSE-LABEL: fptoui_8f32_to_8i32: +; SSE: # BB#0: +; SSE-NEXT: movaps %xmm0, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,1,2,3] +; SSE-NEXT: cvttss2si %xmm0, %rax +; SSE-NEXT: movd %eax, %xmm0 +; SSE-NEXT: movaps %xmm2, %xmm3 +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,1,2,3] +; SSE-NEXT: cvttss2si %xmm3, %rax +; SSE-NEXT: movd %eax, %xmm3 +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: cvttss2si %xmm2, %rax +; SSE-NEXT: movd %eax, %xmm0 +; SSE-NEXT: shufpd {{.*#+}} xmm2 = xmm2[1,0] +; SSE-NEXT: cvttss2si %xmm2, %rax +; SSE-NEXT: movd %eax, %xmm2 +; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1] +; SSE-NEXT: movaps %xmm1, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,1,2,3] +; SSE-NEXT: cvttss2si %xmm2, %rax +; SSE-NEXT: movd %eax, %xmm2 +; SSE-NEXT: movaps %xmm1, %xmm3 +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,1,2,3] +; SSE-NEXT: cvttss2si %xmm3, %rax +; SSE-NEXT: movd %eax, %xmm3 +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] +; SSE-NEXT: cvttss2si %xmm1, %rax +; SSE-NEXT: movd %eax, %xmm2 +; SSE-NEXT: shufpd {{.*#+}} xmm1 = xmm1[1,0] +; SSE-NEXT: cvttss2si %xmm1, %rax +; SSE-NEXT: movd %eax, %xmm1 +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] +; SSE-NEXT: movdqa %xmm2, %xmm1 +; SSE-NEXT: retq +; +; AVX-LABEL: fptoui_8f32_to_8i32: ; AVX: # BB#0: ; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 ; AVX-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3] @@ -658,54 +856,54 @@ define <8 x i32> @fptoui_8vf32(<8 x float> %a) { ret <8 x i32> %cvt } -define <4 x i64> @fptoui_8vf32_i64(<8 x float> %a) { -; SSE2-LABEL: fptoui_8vf32_i64: -; SSE2: # BB#0: -; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero -; SSE2-NEXT: movaps %xmm0, %xmm2 -; SSE2-NEXT: subss %xmm1, %xmm2 -; SSE2-NEXT: cvttss2si %xmm2, %rcx -; SSE2-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000 -; SSE2-NEXT: xorq %rax, %rcx -; SSE2-NEXT: cvttss2si %xmm0, %rdx -; SSE2-NEXT: ucomiss %xmm1, %xmm0 -; SSE2-NEXT: cmovaeq %rcx, %rdx -; SSE2-NEXT: movd %rdx, %xmm2 -; SSE2-NEXT: movaps %xmm0, %xmm3 -; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,1,2,3] -; SSE2-NEXT: movaps %xmm3, %xmm4 -; SSE2-NEXT: subss %xmm1, %xmm4 -; SSE2-NEXT: cvttss2si %xmm4, %rcx -; SSE2-NEXT: xorq %rax, %rcx -; SSE2-NEXT: cvttss2si %xmm3, %rdx -; SSE2-NEXT: ucomiss %xmm1, %xmm3 -; SSE2-NEXT: cmovaeq %rcx, %rdx -; SSE2-NEXT: movd %rdx, %xmm3 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0] -; SSE2-NEXT: movaps %xmm0, %xmm3 -; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,1,2,3] -; SSE2-NEXT: movaps %xmm3, %xmm4 -; SSE2-NEXT: subss %xmm1, %xmm4 -; SSE2-NEXT: cvttss2si %xmm4, %rcx -; SSE2-NEXT: xorq %rax, %rcx -; SSE2-NEXT: cvttss2si %xmm3, %rdx -; SSE2-NEXT: ucomiss %xmm1, %xmm3 -; SSE2-NEXT: cmovaeq %rcx, %rdx -; SSE2-NEXT: movd %rdx, %xmm3 -; SSE2-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] -; SSE2-NEXT: movapd %xmm0, %xmm4 -; SSE2-NEXT: subss %xmm1, %xmm4 -; SSE2-NEXT: cvttss2si %xmm4, %rcx -; SSE2-NEXT: xorq %rax, %rcx -; SSE2-NEXT: cvttss2si %xmm0, %rax -; SSE2-NEXT: ucomiss %xmm1, %xmm0 -; SSE2-NEXT: cmovaeq %rcx, %rax -; SSE2-NEXT: movd %rax, %xmm1 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0] -; SSE2-NEXT: movdqa %xmm2, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: fptoui_8vf32_i64: +define <4 x i64> @fptoui_4f32_to_4i64(<8 x float> %a) { +; SSE-LABEL: fptoui_4f32_to_4i64: +; SSE: # BB#0: +; SSE-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero +; SSE-NEXT: movaps %xmm0, %xmm2 +; SSE-NEXT: subss %xmm1, %xmm2 +; SSE-NEXT: cvttss2si %xmm2, %rcx +; SSE-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000 +; SSE-NEXT: xorq %rax, %rcx +; SSE-NEXT: cvttss2si %xmm0, %rdx +; SSE-NEXT: ucomiss %xmm1, %xmm0 +; SSE-NEXT: cmovaeq %rcx, %rdx +; SSE-NEXT: movd %rdx, %xmm2 +; SSE-NEXT: movaps %xmm0, %xmm3 +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,1,2,3] +; SSE-NEXT: movaps %xmm3, %xmm4 +; SSE-NEXT: subss %xmm1, %xmm4 +; SSE-NEXT: cvttss2si %xmm4, %rcx +; SSE-NEXT: xorq %rax, %rcx +; SSE-NEXT: cvttss2si %xmm3, %rdx +; SSE-NEXT: ucomiss %xmm1, %xmm3 +; SSE-NEXT: cmovaeq %rcx, %rdx +; SSE-NEXT: movd %rdx, %xmm3 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0] +; SSE-NEXT: movaps %xmm0, %xmm3 +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,1,2,3] +; SSE-NEXT: movaps %xmm3, %xmm4 +; SSE-NEXT: subss %xmm1, %xmm4 +; SSE-NEXT: cvttss2si %xmm4, %rcx +; SSE-NEXT: xorq %rax, %rcx +; SSE-NEXT: cvttss2si %xmm3, %rdx +; SSE-NEXT: ucomiss %xmm1, %xmm3 +; SSE-NEXT: cmovaeq %rcx, %rdx +; SSE-NEXT: movd %rdx, %xmm3 +; SSE-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] +; SSE-NEXT: movapd %xmm0, %xmm4 +; SSE-NEXT: subss %xmm1, %xmm4 +; SSE-NEXT: cvttss2si %xmm4, %rcx +; SSE-NEXT: xorq %rax, %rcx +; SSE-NEXT: cvttss2si %xmm0, %rax +; SSE-NEXT: ucomiss %xmm1, %xmm0 +; SSE-NEXT: cmovaeq %rcx, %rax +; SSE-NEXT: movd %rax, %xmm1 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0] +; SSE-NEXT: movdqa %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: fptoui_4f32_to_4i64: ; AVX: # BB#0: ; AVX-NEXT: vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3] ; AVX-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero @@ -744,22 +942,113 @@ define <4 x i64> @fptoui_8vf32_i64(<8 x float> %a) { ; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm3[0],xmm0[0] ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 ; AVX-NEXT: retq - %shuf = shufflevector <8 x float> %a, <8 x float> %a, <4 x i32> + %shuf = shufflevector <8 x float> %a, <8 x float> undef, <4 x i32> %cvt = fptoui <4 x float> %shuf to <4 x i64> ret <4 x i64> %cvt } +define <4 x i64> @fptoui_8f32_to_4i64(<8 x float> %a) { +; SSE-LABEL: fptoui_8f32_to_4i64: +; SSE: # BB#0: +; SSE-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero +; SSE-NEXT: movaps %xmm0, %xmm2 +; SSE-NEXT: subss %xmm1, %xmm2 +; SSE-NEXT: cvttss2si %xmm2, %rcx +; SSE-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000 +; SSE-NEXT: xorq %rax, %rcx +; SSE-NEXT: cvttss2si %xmm0, %rdx +; SSE-NEXT: ucomiss %xmm1, %xmm0 +; SSE-NEXT: cmovaeq %rcx, %rdx +; SSE-NEXT: movd %rdx, %xmm2 +; SSE-NEXT: movaps %xmm0, %xmm3 +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,1,2,3] +; SSE-NEXT: movaps %xmm3, %xmm4 +; SSE-NEXT: subss %xmm1, %xmm4 +; SSE-NEXT: cvttss2si %xmm4, %rcx +; SSE-NEXT: xorq %rax, %rcx +; SSE-NEXT: cvttss2si %xmm3, %rdx +; SSE-NEXT: ucomiss %xmm1, %xmm3 +; SSE-NEXT: cmovaeq %rcx, %rdx +; SSE-NEXT: movd %rdx, %xmm3 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0] +; SSE-NEXT: movaps %xmm0, %xmm3 +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,1,2,3] +; SSE-NEXT: movaps %xmm3, %xmm4 +; SSE-NEXT: subss %xmm1, %xmm4 +; SSE-NEXT: cvttss2si %xmm4, %rcx +; SSE-NEXT: xorq %rax, %rcx +; SSE-NEXT: cvttss2si %xmm3, %rdx +; SSE-NEXT: ucomiss %xmm1, %xmm3 +; SSE-NEXT: cmovaeq %rcx, %rdx +; SSE-NEXT: movd %rdx, %xmm3 +; SSE-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1,0] +; SSE-NEXT: movapd %xmm0, %xmm4 +; SSE-NEXT: subss %xmm1, %xmm4 +; SSE-NEXT: cvttss2si %xmm4, %rcx +; SSE-NEXT: xorq %rax, %rcx +; SSE-NEXT: cvttss2si %xmm0, %rax +; SSE-NEXT: ucomiss %xmm1, %xmm0 +; SSE-NEXT: cmovaeq %rcx, %rax +; SSE-NEXT: movd %rax, %xmm1 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0] +; SSE-NEXT: movdqa %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: fptoui_8f32_to_4i64: +; AVX: # BB#0: +; AVX-NEXT: vpermilps {{.*#+}} xmm2 = xmm0[3,1,2,3] +; AVX-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero +; AVX-NEXT: vsubss %xmm1, %xmm2, %xmm3 +; AVX-NEXT: vcvttss2si %xmm3, %rax +; AVX-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000 +; AVX-NEXT: xorq %rcx, %rax +; AVX-NEXT: vcvttss2si %xmm2, %rdx +; AVX-NEXT: vucomiss %xmm1, %xmm2 +; AVX-NEXT: cmovaeq %rax, %rdx +; AVX-NEXT: vmovq %rdx, %xmm2 +; AVX-NEXT: vpermilpd {{.*#+}} xmm3 = xmm0[1,0] +; AVX-NEXT: vsubss %xmm1, %xmm3, %xmm4 +; AVX-NEXT: vcvttss2si %xmm4, %rax +; AVX-NEXT: xorq %rcx, %rax +; AVX-NEXT: vcvttss2si %xmm3, %rdx +; AVX-NEXT: vucomiss %xmm1, %xmm3 +; AVX-NEXT: cmovaeq %rax, %rdx +; AVX-NEXT: vmovq %rdx, %xmm3 +; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX-NEXT: vsubss %xmm1, %xmm0, %xmm3 +; AVX-NEXT: vcvttss2si %xmm3, %rax +; AVX-NEXT: xorq %rcx, %rax +; AVX-NEXT: vcvttss2si %xmm0, %rdx +; AVX-NEXT: vucomiss %xmm1, %xmm0 +; AVX-NEXT: cmovaeq %rax, %rdx +; AVX-NEXT: vmovq %rdx, %xmm3 +; AVX-NEXT: vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3] +; AVX-NEXT: vsubss %xmm1, %xmm0, %xmm4 +; AVX-NEXT: vcvttss2si %xmm4, %rax +; AVX-NEXT: xorq %rcx, %rax +; AVX-NEXT: vcvttss2si %xmm0, %rcx +; AVX-NEXT: vucomiss %xmm1, %xmm0 +; AVX-NEXT: cmovaeq %rax, %rcx +; AVX-NEXT: vmovq %rcx, %xmm0 +; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm3[0],xmm0[0] +; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX-NEXT: retq + %cvt = fptoui <8 x float> %a to <8 x i64> + %shuf = shufflevector <8 x i64> %cvt, <8 x i64> undef, <4 x i32> + ret <4 x i64> %shuf +} + ; ; Constant Folding ; -define <2 x i64> @fptosi_2vf64c() { -; SSE2-LABEL: fptosi_2vf64c: -; SSE2: # BB#0: -; SSE2-NEXT: movaps {{.*#+}} xmm0 = [1,18446744073709551615] -; SSE2-NEXT: retq +define <2 x i64> @fptosi_2f64_to_2i64_const() { +; SSE-LABEL: fptosi_2f64_to_2i64_const: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [1,18446744073709551615] +; SSE-NEXT: retq ; -; AVX-LABEL: fptosi_2vf64c: +; AVX-LABEL: fptosi_2f64_to_2i64_const: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [1,18446744073709551615] ; AVX-NEXT: retq @@ -767,13 +1056,13 @@ define <2 x i64> @fptosi_2vf64c() { ret <2 x i64> %cvt } -define <4 x i32> @fptosi_2vf64c_i32() { -; SSE2-LABEL: fptosi_2vf64c_i32: -; SSE2: # BB#0: -; SSE2-NEXT: movaps {{.*#+}} xmm0 = <4294967295,1,u,u> -; SSE2-NEXT: retq +define <4 x i32> @fptosi_2f64_to_2i32_const() { +; SSE-LABEL: fptosi_2f64_to_2i32_const: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = <4294967295,1,u,u> +; SSE-NEXT: retq ; -; AVX-LABEL: fptosi_2vf64c_i32: +; AVX-LABEL: fptosi_2f64_to_2i32_const: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} xmm0 = <4294967295,1,u,u> ; AVX-NEXT: retq @@ -782,14 +1071,14 @@ define <4 x i32> @fptosi_2vf64c_i32() { ret <4 x i32> %ext } -define <4 x i64> @fptosi_4vf64c() { -; SSE2-LABEL: fptosi_4vf64c: -; SSE2: # BB#0: -; SSE2-NEXT: movaps {{.*#+}} xmm0 = [1,18446744073709551615] -; SSE2-NEXT: movaps {{.*#+}} xmm1 = [2,18446744073709551613] -; SSE2-NEXT: retq +define <4 x i64> @fptosi_4f64_to_4i64_const() { +; SSE-LABEL: fptosi_4f64_to_4i64_const: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [1,18446744073709551615] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [2,18446744073709551613] +; SSE-NEXT: retq ; -; AVX-LABEL: fptosi_4vf64c: +; AVX-LABEL: fptosi_4f64_to_4i64_const: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [1,18446744073709551615,2,18446744073709551613] ; AVX-NEXT: retq @@ -797,13 +1086,13 @@ define <4 x i64> @fptosi_4vf64c() { ret <4 x i64> %cvt } -define <4 x i32> @fptosi_4vf64c_i32() { -; SSE2-LABEL: fptosi_4vf64c_i32: -; SSE2: # BB#0: -; SSE2-NEXT: movaps {{.*#+}} xmm0 = [4294967295,1,4294967294,3] -; SSE2-NEXT: retq +define <4 x i32> @fptosi_4f64_to_4i32_const() { +; SSE-LABEL: fptosi_4f64_to_4i32_const: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [4294967295,1,4294967294,3] +; SSE-NEXT: retq ; -; AVX-LABEL: fptosi_4vf64c_i32: +; AVX-LABEL: fptosi_4f64_to_4i32_const: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [4294967295,1,4294967294,3] ; AVX-NEXT: retq @@ -811,13 +1100,13 @@ define <4 x i32> @fptosi_4vf64c_i32() { ret <4 x i32> %cvt } -define <2 x i64> @fptoui_2vf64c() { -; SSE2-LABEL: fptoui_2vf64c: -; SSE2: # BB#0: -; SSE2-NEXT: movaps {{.*#+}} xmm0 = [2,4] -; SSE2-NEXT: retq +define <2 x i64> @fptoui_2f64_to_2i64_const() { +; SSE-LABEL: fptoui_2f64_to_2i64_const: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [2,4] +; SSE-NEXT: retq ; -; AVX-LABEL: fptoui_2vf64c: +; AVX-LABEL: fptoui_2f64_to_2i64_const: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [2,4] ; AVX-NEXT: retq @@ -825,13 +1114,13 @@ define <2 x i64> @fptoui_2vf64c() { ret <2 x i64> %cvt } -define <4 x i32> @fptoui_2vf64c_i32(<2 x double> %a) { -; SSE2-LABEL: fptoui_2vf64c_i32: -; SSE2: # BB#0: -; SSE2-NEXT: movaps {{.*#+}} xmm0 = <2,4,u,u> -; SSE2-NEXT: retq +define <4 x i32> @fptoui_2f64_to_2i32_const(<2 x double> %a) { +; SSE-LABEL: fptoui_2f64_to_2i32_const: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = <2,4,u,u> +; SSE-NEXT: retq ; -; AVX-LABEL: fptoui_2vf64c_i32: +; AVX-LABEL: fptoui_2f64_to_2i32_const: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} xmm0 = <2,4,u,u> ; AVX-NEXT: retq @@ -840,14 +1129,14 @@ define <4 x i32> @fptoui_2vf64c_i32(<2 x double> %a) { ret <4 x i32> %ext } -define <4 x i64> @fptoui_4vf64c(<4 x double> %a) { -; SSE2-LABEL: fptoui_4vf64c: -; SSE2: # BB#0: -; SSE2-NEXT: movaps {{.*#+}} xmm0 = [2,4] -; SSE2-NEXT: movaps {{.*#+}} xmm1 = [6,8] -; SSE2-NEXT: retq +define <4 x i64> @fptoui_4f64_to_4i64_const(<4 x double> %a) { +; SSE-LABEL: fptoui_4f64_to_4i64_const: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [2,4] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [6,8] +; SSE-NEXT: retq ; -; AVX-LABEL: fptoui_4vf64c: +; AVX-LABEL: fptoui_4f64_to_4i64_const: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [2,4,6,8] ; AVX-NEXT: retq @@ -855,13 +1144,13 @@ define <4 x i64> @fptoui_4vf64c(<4 x double> %a) { ret <4 x i64> %cvt } -define <4 x i32> @fptoui_4vf64c_i32(<4 x double> %a) { -; SSE2-LABEL: fptoui_4vf64c_i32: -; SSE2: # BB#0: -; SSE2-NEXT: movaps {{.*#+}} xmm0 = [2,4,6,8] -; SSE2-NEXT: retq +define <4 x i32> @fptoui_4f64_to_4i32_const(<4 x double> %a) { +; SSE-LABEL: fptoui_4f64_to_4i32_const: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [2,4,6,8] +; SSE-NEXT: retq ; -; AVX-LABEL: fptoui_4vf64c_i32: +; AVX-LABEL: fptoui_4f64_to_4i32_const: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [2,4,6,8] ; AVX-NEXT: retq @@ -869,13 +1158,13 @@ define <4 x i32> @fptoui_4vf64c_i32(<4 x double> %a) { ret <4 x i32> %cvt } -define <4 x i32> @fptosi_4vf32c() { -; SSE2-LABEL: fptosi_4vf32c: -; SSE2: # BB#0: -; SSE2-NEXT: movaps {{.*#+}} xmm0 = [1,4294967295,2,3] -; SSE2-NEXT: retq +define <4 x i32> @fptosi_4f32_to_4i32_const() { +; SSE-LABEL: fptosi_4f32_to_4i32_const: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [1,4294967295,2,3] +; SSE-NEXT: retq ; -; AVX-LABEL: fptosi_4vf32c: +; AVX-LABEL: fptosi_4f32_to_4i32_const: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [1,4294967295,2,3] ; AVX-NEXT: retq @@ -883,14 +1172,14 @@ define <4 x i32> @fptosi_4vf32c() { ret <4 x i32> %cvt } -define <4 x i64> @fptosi_4vf32c_i64() { -; SSE2-LABEL: fptosi_4vf32c_i64: -; SSE2: # BB#0: -; SSE2-NEXT: movaps {{.*#+}} xmm0 = [1,18446744073709551615] -; SSE2-NEXT: movaps {{.*#+}} xmm1 = [2,3] -; SSE2-NEXT: retq +define <4 x i64> @fptosi_4f32_to_4i64_const() { +; SSE-LABEL: fptosi_4f32_to_4i64_const: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [1,18446744073709551615] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [2,3] +; SSE-NEXT: retq ; -; AVX-LABEL: fptosi_4vf32c_i64: +; AVX-LABEL: fptosi_4f32_to_4i64_const: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [1,18446744073709551615,2,3] ; AVX-NEXT: retq @@ -898,14 +1187,14 @@ define <4 x i64> @fptosi_4vf32c_i64() { ret <4 x i64> %cvt } -define <8 x i32> @fptosi_8vf32c(<8 x float> %a) { -; SSE2-LABEL: fptosi_8vf32c: -; SSE2: # BB#0: -; SSE2-NEXT: movaps {{.*#+}} xmm0 = [1,4294967295,2,3] -; SSE2-NEXT: movaps {{.*#+}} xmm1 = [6,4294967288,2,4294967295] -; SSE2-NEXT: retq +define <8 x i32> @fptosi_8f32_to_8i32_const(<8 x float> %a) { +; SSE-LABEL: fptosi_8f32_to_8i32_const: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [1,4294967295,2,3] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [6,4294967288,2,4294967295] +; SSE-NEXT: retq ; -; AVX-LABEL: fptosi_8vf32c: +; AVX-LABEL: fptosi_8f32_to_8i32_const: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [1,4294967295,2,3,6,4294967288,2,4294967295] ; AVX-NEXT: retq @@ -913,13 +1202,13 @@ define <8 x i32> @fptosi_8vf32c(<8 x float> %a) { ret <8 x i32> %cvt } -define <4 x i32> @fptoui_4vf32c(<4 x float> %a) { -; SSE2-LABEL: fptoui_4vf32c: -; SSE2: # BB#0: -; SSE2-NEXT: movaps {{.*#+}} xmm0 = [1,2,4,6] -; SSE2-NEXT: retq +define <4 x i32> @fptoui_4f32_to_4i32_const(<4 x float> %a) { +; SSE-LABEL: fptoui_4f32_to_4i32_const: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [1,2,4,6] +; SSE-NEXT: retq ; -; AVX-LABEL: fptoui_4vf32c: +; AVX-LABEL: fptoui_4f32_to_4i32_const: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [1,2,4,6] ; AVX-NEXT: retq @@ -927,14 +1216,14 @@ define <4 x i32> @fptoui_4vf32c(<4 x float> %a) { ret <4 x i32> %cvt } -define <4 x i64> @fptoui_4vf32c_i64() { -; SSE2-LABEL: fptoui_4vf32c_i64: -; SSE2: # BB#0: -; SSE2-NEXT: movaps {{.*#+}} xmm0 = [1,2] -; SSE2-NEXT: movaps {{.*#+}} xmm1 = [4,8] -; SSE2-NEXT: retq +define <4 x i64> @fptoui_4f32_to_4i64_const() { +; SSE-LABEL: fptoui_4f32_to_4i64_const: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [1,2] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [4,8] +; SSE-NEXT: retq ; -; AVX-LABEL: fptoui_4vf32c_i64: +; AVX-LABEL: fptoui_4f32_to_4i64_const: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [1,2,4,8] ; AVX-NEXT: retq @@ -942,14 +1231,14 @@ define <4 x i64> @fptoui_4vf32c_i64() { ret <4 x i64> %cvt } -define <8 x i32> @fptoui_8vf32c(<8 x float> %a) { -; SSE2-LABEL: fptoui_8vf32c: -; SSE2: # BB#0: -; SSE2-NEXT: movaps {{.*#+}} xmm0 = [1,2,4,6] -; SSE2-NEXT: movaps {{.*#+}} xmm1 = [8,6,4,1] -; SSE2-NEXT: retq +define <8 x i32> @fptoui_8f32_to_8i32_const(<8 x float> %a) { +; SSE-LABEL: fptoui_8f32_to_8i32_const: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [1,2,4,6] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [8,6,4,1] +; SSE-NEXT: retq ; -; AVX-LABEL: fptoui_8vf32c: +; AVX-LABEL: fptoui_8f32_to_8i32_const: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [1,2,4,6,8,6,4,1] ; AVX-NEXT: retq diff --git a/test/CodeGen/X86/vec_insert-5.ll b/test/CodeGen/X86/vec_insert-5.ll index 4018a21090e7..14b57e76dc8f 100644 --- a/test/CodeGen/X86/vec_insert-5.ll +++ b/test/CodeGen/X86/vec_insert-5.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -march=x86 -mattr=+sse2,+ssse3 | FileCheck %s ; There are no MMX operations in @t1 diff --git a/test/CodeGen/X86/vec_int_to_fp.ll b/test/CodeGen/X86/vec_int_to_fp.ll index 4a3d08813904..fd98791815e7 100644 --- a/test/CodeGen/X86/vec_int_to_fp.ll +++ b/test/CodeGen/X86/vec_int_to_fp.ll @@ -11,20 +11,20 @@ ; Signed Integer to Double ; -define <2 x double> @sitofp_2vf64(<2 x i64> %a) { -; SSE2-LABEL: sitofp_2vf64: -; SSE2: # BB#0: -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: cvtsi2sdq %rax, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: xorps %xmm0, %xmm0 -; SSE2-NEXT: cvtsi2sdq %rax, %xmm0 -; SSE2-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0] -; SSE2-NEXT: movapd %xmm1, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: sitofp_2vf64: +define <2 x double> @sitofp_2i64_to_2f64(<2 x i64> %a) { +; SSE-LABEL: sitofp_2i64_to_2f64: +; SSE: # BB#0: +; SSE-NEXT: movd %xmm0, %rax +; SSE-NEXT: cvtsi2sdq %rax, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; SSE-NEXT: movd %xmm0, %rax +; SSE-NEXT: xorps %xmm0, %xmm0 +; SSE-NEXT: cvtsi2sdq %rax, %xmm0 +; SSE-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; SSE-NEXT: movapd %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: sitofp_2i64_to_2f64: ; AVX: # BB#0: ; AVX-NEXT: vpextrq $1, %xmm0, %rax ; AVX-NEXT: vcvtsi2sdq %rax, %xmm0, %xmm1 @@ -37,13 +37,13 @@ define <2 x double> @sitofp_2vf64(<2 x i64> %a) { ret <2 x double> %cvt } -define <2 x double> @sitofp_2vf64_i32(<4 x i32> %a) { -; SSE2-LABEL: sitofp_2vf64_i32: -; SSE2: # BB#0: -; SSE2-NEXT: cvtdq2pd %xmm0, %xmm0 -; SSE2-NEXT: retq +define <2 x double> @sitofp_2i32_to_2f64(<4 x i32> %a) { +; SSE-LABEL: sitofp_2i32_to_2f64: +; SSE: # BB#0: +; SSE-NEXT: cvtdq2pd %xmm0, %xmm0 +; SSE-NEXT: retq ; -; AVX-LABEL: sitofp_2vf64_i32: +; AVX-LABEL: sitofp_2i32_to_2f64: ; AVX: # BB#0: ; AVX-NEXT: vcvtdq2pd %xmm0, %xmm0 ; AVX-NEXT: retq @@ -52,15 +52,31 @@ define <2 x double> @sitofp_2vf64_i32(<4 x i32> %a) { ret <2 x double> %cvt } -define <2 x double> @sitofp_2vf64_i16(<8 x i16> %a) { -; SSE2-LABEL: sitofp_2vf64_i16: -; SSE2: # BB#0: -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] -; SSE2-NEXT: psrad $16, %xmm0 -; SSE2-NEXT: cvtdq2pd %xmm0, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: sitofp_2vf64_i16: +define <2 x double> @sitofp_4i32_to_2f64(<4 x i32> %a) { +; SSE-LABEL: sitofp_4i32_to_2f64: +; SSE: # BB#0: +; SSE-NEXT: cvtdq2pd %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: sitofp_4i32_to_2f64: +; AVX: # BB#0: +; AVX-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX-NEXT: vzeroupper +; AVX-NEXT: retq + %cvt = sitofp <4 x i32> %a to <4 x double> + %shuf = shufflevector <4 x double> %cvt, <4 x double> undef, <2 x i32> + ret <2 x double> %shuf +} + +define <2 x double> @sitofp_2i16_to_2f64(<8 x i16> %a) { +; SSE-LABEL: sitofp_2i16_to_2f64: +; SSE: # BB#0: +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE-NEXT: psrad $16, %xmm0 +; SSE-NEXT: cvtdq2pd %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: sitofp_2i16_to_2f64: ; AVX: # BB#0: ; AVX-NEXT: vpmovsxwd %xmm0, %xmm0 ; AVX-NEXT: vcvtdq2pd %xmm0, %xmm0 @@ -70,16 +86,42 @@ define <2 x double> @sitofp_2vf64_i16(<8 x i16> %a) { ret <2 x double> %cvt } -define <2 x double> @sitofp_2vf64_i8(<16 x i8> %a) { -; SSE2-LABEL: sitofp_2vf64_i8: -; SSE2: # BB#0: -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] -; SSE2-NEXT: psrad $24, %xmm0 -; SSE2-NEXT: cvtdq2pd %xmm0, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: sitofp_2vf64_i8: +define <2 x double> @sitofp_8i16_to_2f64(<8 x i16> %a) { +; SSE-LABEL: sitofp_8i16_to_2f64: +; SSE: # BB#0: +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE-NEXT: psrad $16, %xmm0 +; SSE-NEXT: cvtdq2pd %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: sitofp_8i16_to_2f64: +; AVX1: # BB#0: +; AVX1-NEXT: vpmovsxwd %xmm0, %xmm0 +; AVX1-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: sitofp_8i16_to_2f64: +; AVX2: # BB#0: +; AVX2-NEXT: vpmovsxwd %xmm0, %ymm0 +; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq + %cvt = sitofp <8 x i16> %a to <8 x double> + %shuf = shufflevector <8 x double> %cvt, <8 x double> undef, <2 x i32> + ret <2 x double> %shuf +} + +define <2 x double> @sitofp_2i8_to_2f64(<16 x i8> %a) { +; SSE-LABEL: sitofp_2i8_to_2f64: +; SSE: # BB#0: +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE-NEXT: psrad $24, %xmm0 +; SSE-NEXT: cvtdq2pd %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: sitofp_2i8_to_2f64: ; AVX: # BB#0: ; AVX-NEXT: vpmovsxbd %xmm0, %xmm0 ; AVX-NEXT: vcvtdq2pd %xmm0, %xmm0 @@ -89,28 +131,56 @@ define <2 x double> @sitofp_2vf64_i8(<16 x i8> %a) { ret <2 x double> %cvt } -define <4 x double> @sitofp_4vf64(<4 x i64> %a) { -; SSE2-LABEL: sitofp_4vf64: -; SSE2: # BB#0: -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: cvtsi2sdq %rax, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: xorps %xmm0, %xmm0 -; SSE2-NEXT: cvtsi2sdq %rax, %xmm0 -; SSE2-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0] -; SSE2-NEXT: movd %xmm1, %rax -; SSE2-NEXT: cvtsi2sdq %rax, %xmm3 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: xorps %xmm0, %xmm0 -; SSE2-NEXT: cvtsi2sdq %rax, %xmm0 -; SSE2-NEXT: unpcklpd {{.*#+}} xmm3 = xmm3[0],xmm0[0] -; SSE2-NEXT: movapd %xmm2, %xmm0 -; SSE2-NEXT: movapd %xmm3, %xmm1 -; SSE2-NEXT: retq -; -; AVX1-LABEL: sitofp_4vf64: +define <2 x double> @sitofp_16i8_to_2f64(<16 x i8> %a) { +; SSE-LABEL: sitofp_16i8_to_2f64: +; SSE: # BB#0: +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE-NEXT: psrad $24, %xmm0 +; SSE-NEXT: cvtdq2pd %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: sitofp_16i8_to_2f64: +; AVX1: # BB#0: +; AVX1-NEXT: vpmovsxbd %xmm0, %xmm0 +; AVX1-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: sitofp_16i8_to_2f64: +; AVX2: # BB#0: +; AVX2-NEXT: vpmovsxbw %xmm0, %ymm0 +; AVX2-NEXT: vpmovsxwd %xmm0, %ymm0 +; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq + %cvt = sitofp <16 x i8> %a to <16 x double> + %shuf = shufflevector <16 x double> %cvt, <16 x double> undef, <2 x i32> + ret <2 x double> %shuf +} + +define <4 x double> @sitofp_4i64_to_4f64(<4 x i64> %a) { +; SSE-LABEL: sitofp_4i64_to_4f64: +; SSE: # BB#0: +; SSE-NEXT: movd %xmm0, %rax +; SSE-NEXT: cvtsi2sdq %rax, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; SSE-NEXT: movd %xmm0, %rax +; SSE-NEXT: xorps %xmm0, %xmm0 +; SSE-NEXT: cvtsi2sdq %rax, %xmm0 +; SSE-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0] +; SSE-NEXT: movd %xmm1, %rax +; SSE-NEXT: cvtsi2sdq %rax, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] +; SSE-NEXT: movd %xmm0, %rax +; SSE-NEXT: xorps %xmm0, %xmm0 +; SSE-NEXT: cvtsi2sdq %rax, %xmm0 +; SSE-NEXT: unpcklpd {{.*#+}} xmm3 = xmm3[0],xmm0[0] +; SSE-NEXT: movapd %xmm2, %xmm0 +; SSE-NEXT: movapd %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: sitofp_4i64_to_4f64: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 ; AVX1-NEXT: vpextrq $1, %xmm1, %rax @@ -127,7 +197,7 @@ define <4 x double> @sitofp_4vf64(<4 x i64> %a) { ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; -; AVX2-LABEL: sitofp_4vf64: +; AVX2-LABEL: sitofp_4i64_to_4f64: ; AVX2: # BB#0: ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 ; AVX2-NEXT: vpextrq $1, %xmm1, %rax @@ -147,16 +217,16 @@ define <4 x double> @sitofp_4vf64(<4 x i64> %a) { ret <4 x double> %cvt } -define <4 x double> @sitofp_4vf64_i32(<4 x i32> %a) { -; SSE2-LABEL: sitofp_4vf64_i32: -; SSE2: # BB#0: -; SSE2-NEXT: cvtdq2pd %xmm0, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE2-NEXT: cvtdq2pd %xmm0, %xmm1 -; SSE2-NEXT: movaps %xmm2, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: sitofp_4vf64_i32: +define <4 x double> @sitofp_4i32_to_4f64(<4 x i32> %a) { +; SSE-LABEL: sitofp_4i32_to_4f64: +; SSE: # BB#0: +; SSE-NEXT: cvtdq2pd %xmm0, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; SSE-NEXT: cvtdq2pd %xmm0, %xmm1 +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: sitofp_4i32_to_4f64: ; AVX: # BB#0: ; AVX-NEXT: vcvtdq2pd %xmm0, %ymm0 ; AVX-NEXT: retq @@ -164,17 +234,17 @@ define <4 x double> @sitofp_4vf64_i32(<4 x i32> %a) { ret <4 x double> %cvt } -define <4 x double> @sitofp_4vf64_i16(<8 x i16> %a) { -; SSE2-LABEL: sitofp_4vf64_i16: -; SSE2: # BB#0: -; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] -; SSE2-NEXT: psrad $16, %xmm1 -; SSE2-NEXT: cvtdq2pd %xmm1, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] -; SSE2-NEXT: cvtdq2pd %xmm1, %xmm1 -; SSE2-NEXT: retq -; -; AVX-LABEL: sitofp_4vf64_i16: +define <4 x double> @sitofp_4i16_to_4f64(<8 x i16> %a) { +; SSE-LABEL: sitofp_4i16_to_4f64: +; SSE: # BB#0: +; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSE-NEXT: psrad $16, %xmm1 +; SSE-NEXT: cvtdq2pd %xmm1, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] +; SSE-NEXT: cvtdq2pd %xmm1, %xmm1 +; SSE-NEXT: retq +; +; AVX-LABEL: sitofp_4i16_to_4f64: ; AVX: # BB#0: ; AVX-NEXT: vpmovsxwd %xmm0, %xmm0 ; AVX-NEXT: vcvtdq2pd %xmm0, %ymm0 @@ -184,18 +254,44 @@ define <4 x double> @sitofp_4vf64_i16(<8 x i16> %a) { ret <4 x double> %cvt } -define <4 x double> @sitofp_4vf64_i8(<16 x i8> %a) { -; SSE2-LABEL: sitofp_4vf64_i8: -; SSE2: # BB#0: -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] -; SSE2-NEXT: psrad $24, %xmm1 -; SSE2-NEXT: cvtdq2pd %xmm1, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] -; SSE2-NEXT: cvtdq2pd %xmm1, %xmm1 -; SSE2-NEXT: retq -; -; AVX-LABEL: sitofp_4vf64_i8: +define <4 x double> @sitofp_8i16_to_4f64(<8 x i16> %a) { +; SSE-LABEL: sitofp_8i16_to_4f64: +; SSE: # BB#0: +; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSE-NEXT: psrad $16, %xmm1 +; SSE-NEXT: cvtdq2pd %xmm1, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] +; SSE-NEXT: cvtdq2pd %xmm1, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: sitofp_8i16_to_4f64: +; AVX1: # BB#0: +; AVX1-NEXT: vpmovsxwd %xmm0, %xmm0 +; AVX1-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: sitofp_8i16_to_4f64: +; AVX2: # BB#0: +; AVX2-NEXT: vpmovsxwd %xmm0, %ymm0 +; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX2-NEXT: retq + %cvt = sitofp <8 x i16> %a to <8 x double> + %shuf = shufflevector <8 x double> %cvt, <8 x double> undef, <4 x i32> + ret <4 x double> %shuf +} + +define <4 x double> @sitofp_4i8_to_4f64(<16 x i8> %a) { +; SSE-LABEL: sitofp_4i8_to_4f64: +; SSE: # BB#0: +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSE-NEXT: psrad $24, %xmm1 +; SSE-NEXT: cvtdq2pd %xmm1, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] +; SSE-NEXT: cvtdq2pd %xmm1, %xmm1 +; SSE-NEXT: retq +; +; AVX-LABEL: sitofp_4i8_to_4f64: ; AVX: # BB#0: ; AVX-NEXT: vpmovsxbd %xmm0, %xmm0 ; AVX-NEXT: vcvtdq2pd %xmm0, %ymm0 @@ -205,28 +301,56 @@ define <4 x double> @sitofp_4vf64_i8(<16 x i8> %a) { ret <4 x double> %cvt } +define <4 x double> @sitofp_16i8_to_4f64(<16 x i8> %a) { +; SSE-LABEL: sitofp_16i8_to_4f64: +; SSE: # BB#0: +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSE-NEXT: psrad $24, %xmm1 +; SSE-NEXT: cvtdq2pd %xmm1, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] +; SSE-NEXT: cvtdq2pd %xmm1, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: sitofp_16i8_to_4f64: +; AVX1: # BB#0: +; AVX1-NEXT: vpmovsxbd %xmm0, %xmm0 +; AVX1-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: sitofp_16i8_to_4f64: +; AVX2: # BB#0: +; AVX2-NEXT: vpmovsxbw %xmm0, %ymm0 +; AVX2-NEXT: vpmovsxwd %xmm0, %ymm0 +; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX2-NEXT: retq + %cvt = sitofp <16 x i8> %a to <16 x double> + %shuf = shufflevector <16 x double> %cvt, <16 x double> undef, <4 x i32> + ret <4 x double> %shuf +} + ; ; Unsigned Integer to Double ; -define <2 x double> @uitofp_2vf64(<2 x i64> %a) { -; SSE2-LABEL: uitofp_2vf64: -; SSE2: # BB#0: -; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [1127219200,1160773632,0,0] -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1] -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] -; SSE2-NEXT: movapd {{.*#+}} xmm3 = [4.503600e+15,1.934281e+25] -; SSE2-NEXT: subpd %xmm3, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1] -; SSE2-NEXT: addpd %xmm4, %xmm0 -; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; SSE2-NEXT: subpd %xmm3, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1] -; SSE2-NEXT: addpd %xmm2, %xmm1 -; SSE2-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0] -; SSE2-NEXT: retq -; -; AVX-LABEL: uitofp_2vf64: +define <2 x double> @uitofp_2i64_to_2f64(<2 x i64> %a) { +; SSE-LABEL: uitofp_2i64_to_2f64: +; SSE: # BB#0: +; SSE-NEXT: movdqa {{.*#+}} xmm1 = [1127219200,1160773632,0,0] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1] +; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE-NEXT: movapd {{.*#+}} xmm3 = [4.503600e+15,1.934281e+25] +; SSE-NEXT: subpd %xmm3, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1] +; SSE-NEXT: addpd %xmm4, %xmm0 +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE-NEXT: subpd %xmm3, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1] +; SSE-NEXT: addpd %xmm2, %xmm1 +; SSE-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE-NEXT: retq +; +; AVX-LABEL: uitofp_2i64_to_2f64: ; AVX: # BB#0: ; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [1127219200,1160773632,0,0] ; AVX-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] @@ -243,26 +367,26 @@ define <2 x double> @uitofp_2vf64(<2 x i64> %a) { ret <2 x double> %cvt } -define <2 x double> @uitofp_2vf64_i32(<4 x i32> %a) { -; SSE2-LABEL: uitofp_2vf64_i32: -; SSE2: # BB#0: -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] -; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [1127219200,1160773632,0,0] -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1] -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] -; SSE2-NEXT: movapd {{.*#+}} xmm3 = [4.503600e+15,1.934281e+25] -; SSE2-NEXT: subpd %xmm3, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1] -; SSE2-NEXT: addpd %xmm4, %xmm0 -; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; SSE2-NEXT: subpd %xmm3, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1] -; SSE2-NEXT: addpd %xmm2, %xmm1 -; SSE2-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0] -; SSE2-NEXT: retq -; -; AVX-LABEL: uitofp_2vf64_i32: +define <2 x double> @uitofp_2i32_to_2f64(<4 x i32> %a) { +; SSE-LABEL: uitofp_2i32_to_2f64: +; SSE: # BB#0: +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE-NEXT: movdqa {{.*#+}} xmm1 = [1127219200,1160773632,0,0] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1] +; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE-NEXT: movapd {{.*#+}} xmm3 = [4.503600e+15,1.934281e+25] +; SSE-NEXT: subpd %xmm3, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1] +; SSE-NEXT: addpd %xmm4, %xmm0 +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE-NEXT: subpd %xmm3, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1] +; SSE-NEXT: addpd %xmm2, %xmm1 +; SSE-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE-NEXT: retq +; +; AVX-LABEL: uitofp_2i32_to_2f64: ; AVX: # BB#0: ; AVX-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero ; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [1127219200,1160773632,0,0] @@ -281,21 +405,64 @@ define <2 x double> @uitofp_2vf64_i32(<4 x i32> %a) { ret <2 x double> %cvt } -define <2 x double> @uitofp_2vf64_i16(<8 x i16> %a) { -; SSE2-LABEL: uitofp_2vf64_i16: -; SSE2: # BB#0: -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] -; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,7] -; SSE2-NEXT: pand .LCPI10_0(%rip), %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; SSE2-NEXT: cvtdq2pd %xmm0, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: uitofp_2vf64_i16: +define <2 x double> @uitofp_4i32_to_2f64(<4 x i32> %a) { +; SSE-LABEL: uitofp_4i32_to_2f64: +; SSE: # BB#0: +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE-NEXT: movdqa {{.*#+}} xmm1 = [1127219200,1160773632,0,0] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1] +; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE-NEXT: movapd {{.*#+}} xmm3 = [4.503600e+15,1.934281e+25] +; SSE-NEXT: subpd %xmm3, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1] +; SSE-NEXT: addpd %xmm4, %xmm0 +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE-NEXT: subpd %xmm3, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1] +; SSE-NEXT: addpd %xmm2, %xmm1 +; SSE-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE-NEXT: retq +; +; AVX1-LABEL: uitofp_4i32_to_2f64: +; AVX1: # BB#0: +; AVX1-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm1 +; AVX1-NEXT: vcvtdq2pd %xmm1, %ymm1 +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX1-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX1-NEXT: vmulpd {{.*}}(%rip), %ymm0, %ymm0 +; AVX1-NEXT: vaddpd %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: uitofp_4i32_to_2f64: +; AVX2: # BB#0: +; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX2-NEXT: vcvtdq2pd %xmm1, %ymm1 +; AVX2-NEXT: vbroadcastsd {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vmulpd %ymm2, %ymm1, %ymm1 +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %xmm2 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX2-NEXT: vaddpd %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq + %cvt = uitofp <4 x i32> %a to <4 x double> + %shuf = shufflevector <4 x double> %cvt, <4 x double> undef, <2 x i32> + ret <2 x double> %shuf +} + +define <2 x double> @uitofp_2i16_to_2f64(<8 x i16> %a) { +; SSE-LABEL: uitofp_2i16_to_2f64: +; SSE: # BB#0: +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE-NEXT: cvtdq2pd %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: uitofp_2i16_to_2f64: ; AVX: # BB#0: -; AVX-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero -; AVX-NEXT: vpand .LCPI10_0(%rip), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; AVX-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero ; AVX-NEXT: vcvtdq2pd %xmm0, %xmm0 ; AVX-NEXT: retq %shuf = shufflevector <8 x i16> %a, <8 x i16> undef, <2 x i32> @@ -303,22 +470,44 @@ define <2 x double> @uitofp_2vf64_i16(<8 x i16> %a) { ret <2 x double> %cvt } -define <2 x double> @uitofp_2vf64_i8(<16 x i8> %a) { -; SSE2-LABEL: uitofp_2vf64_i8: -; SSE2: # BB#0: -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0,0,1,1] -; SSE2-NEXT: pand .LCPI11_0(%rip), %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; SSE2-NEXT: cvtdq2pd %xmm0, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: uitofp_2vf64_i8: +define <2 x double> @uitofp_8i16_to_2f64(<8 x i16> %a) { +; SSE-LABEL: uitofp_8i16_to_2f64: +; SSE: # BB#0: +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE-NEXT: cvtdq2pd %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: uitofp_8i16_to_2f64: +; AVX1: # BB#0: +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX1-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: uitofp_8i16_to_2f64: +; AVX2: # BB#0: +; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq + %cvt = uitofp <8 x i16> %a to <8 x double> + %shuf = shufflevector <8 x double> %cvt, <8 x double> undef, <2 x i32> + ret <2 x double> %shuf +} + +define <2 x double> @uitofp_2i8_to_2f64(<16 x i8> %a) { +; SSE-LABEL: uitofp_2i8_to_2f64: +; SSE: # BB#0: +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE-NEXT: cvtdq2pd %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: uitofp_2i8_to_2f64: ; AVX: # BB#0: -; AVX-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpand .LCPI11_0(%rip), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; AVX-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero ; AVX-NEXT: vcvtdq2pd %xmm0, %xmm0 ; AVX-NEXT: retq %shuf = shufflevector <16 x i8> %a, <16 x i8> undef, <2 x i32> @@ -326,34 +515,62 @@ define <2 x double> @uitofp_2vf64_i8(<16 x i8> %a) { ret <2 x double> %cvt } -define <4 x double> @uitofp_4vf64(<4 x i64> %a) { -; SSE2-LABEL: uitofp_4vf64: -; SSE2: # BB#0: -; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [1127219200,1160773632,0,0] -; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1] -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] -; SSE2-NEXT: movapd {{.*#+}} xmm4 = [4.503600e+15,1.934281e+25] -; SSE2-NEXT: subpd %xmm4, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm0[2,3,0,1] -; SSE2-NEXT: addpd %xmm5, %xmm0 -; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; SSE2-NEXT: subpd %xmm4, %xmm3 -; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm3[2,3,0,1] -; SSE2-NEXT: addpd %xmm3, %xmm5 -; SSE2-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm5[0] -; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1] -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] -; SSE2-NEXT: subpd %xmm4, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm1[2,3,0,1] -; SSE2-NEXT: addpd %xmm5, %xmm1 -; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; SSE2-NEXT: subpd %xmm4, %xmm3 -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1] -; SSE2-NEXT: addpd %xmm3, %xmm2 -; SSE2-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0] -; SSE2-NEXT: retq -; -; AVX1-LABEL: uitofp_4vf64: +define <2 x double> @uitofp_16i8_to_2f64(<16 x i8> %a) { +; SSE-LABEL: uitofp_16i8_to_2f64: +; SSE: # BB#0: +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE-NEXT: cvtdq2pd %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: uitofp_16i8_to_2f64: +; AVX1: # BB#0: +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX1-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: uitofp_16i8_to_2f64: +; AVX2: # BB#0: +; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq + %cvt = uitofp <16 x i8> %a to <16 x double> + %shuf = shufflevector <16 x double> %cvt, <16 x double> undef, <2 x i32> + ret <2 x double> %shuf +} + +define <4 x double> @uitofp_4i64_to_4f64(<4 x i64> %a) { +; SSE-LABEL: uitofp_4i64_to_4f64: +; SSE: # BB#0: +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [1127219200,1160773632,0,0] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1] +; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSE-NEXT: movapd {{.*#+}} xmm4 = [4.503600e+15,1.934281e+25] +; SSE-NEXT: subpd %xmm4, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm0[2,3,0,1] +; SSE-NEXT: addpd %xmm5, %xmm0 +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] +; SSE-NEXT: subpd %xmm4, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm3[2,3,0,1] +; SSE-NEXT: addpd %xmm3, %xmm5 +; SSE-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm5[0] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1] +; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; SSE-NEXT: subpd %xmm4, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[2,3,0,1] +; SSE-NEXT: addpd %xmm5, %xmm1 +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] +; SSE-NEXT: subpd %xmm4, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1] +; SSE-NEXT: addpd %xmm3, %xmm2 +; SSE-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0] +; SSE-NEXT: retq +; +; AVX1-LABEL: uitofp_4i64_to_4f64: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 ; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [1127219200,1160773632,0,0] @@ -377,7 +594,7 @@ define <4 x double> @uitofp_4vf64(<4 x i64> %a) { ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; -; AVX2-LABEL: uitofp_4vf64: +; AVX2-LABEL: uitofp_4i64_to_4f64: ; AVX2: # BB#0: ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [1127219200,1160773632,0,0] @@ -404,54 +621,54 @@ define <4 x double> @uitofp_4vf64(<4 x i64> %a) { ret <4 x double> %cvt } -define <4 x double> @uitofp_4vf64_i32(<4 x i32> %a) { -; SSE2-LABEL: uitofp_4vf64_i32: -; SSE2: # BB#0: -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,2,3,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] -; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1127219200,1160773632,0,0] -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1] -; SSE2-NEXT: movapd {{.*#+}} xmm4 = [4.503600e+15,1.934281e+25] -; SSE2-NEXT: subpd %xmm4, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm0[2,3,0,1] -; SSE2-NEXT: addpd %xmm5, %xmm0 -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] -; SSE2-NEXT: subpd %xmm4, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm1[2,3,0,1] -; SSE2-NEXT: addpd %xmm1, %xmm5 -; SSE2-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm5[0] -; SSE2-NEXT: pand .LCPI13_2(%rip), %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm2[2,3,0,1] -; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] -; SSE2-NEXT: subpd %xmm4, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1] -; SSE2-NEXT: addpd %xmm2, %xmm1 -; SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1] -; SSE2-NEXT: subpd %xmm4, %xmm5 -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm5[2,3,0,1] -; SSE2-NEXT: addpd %xmm5, %xmm2 -; SSE2-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0] -; SSE2-NEXT: retq -; -; AVX1-LABEL: uitofp_4vf64_i32: +define <4 x double> @uitofp_4i32_to_4f64(<4 x i32> %a) { +; SSE-LABEL: uitofp_4i32_to_4f64: +; SSE: # BB#0: +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE-NEXT: movdqa {{.*#+}} xmm3 = [1127219200,1160773632,0,0] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,0,1] +; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1] +; SSE-NEXT: movapd {{.*#+}} xmm5 = [4.503600e+15,1.934281e+25] +; SSE-NEXT: subpd %xmm5, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm0[2,3,0,1] +; SSE-NEXT: addpd %xmm6, %xmm0 +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: subpd %xmm5, %xmm4 +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm4[2,3,0,1] +; SSE-NEXT: addpd %xmm4, %xmm6 +; SSE-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm6[0] +; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,3,0,1] +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] +; SSE-NEXT: subpd %xmm5, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1] +; SSE-NEXT: addpd %xmm2, %xmm1 +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: subpd %xmm5, %xmm4 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[2,3,0,1] +; SSE-NEXT: addpd %xmm4, %xmm2 +; SSE-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm2[0] +; SSE-NEXT: retq +; +; AVX1-LABEL: uitofp_4i32_to_4f64: ; AVX1: # BB#0: -; AVX1-NEXT: vpand .LCPI13_0(%rip), %xmm0, %xmm1 +; AVX1-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm1 ; AVX1-NEXT: vcvtdq2pd %xmm1, %ymm1 ; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0 ; AVX1-NEXT: vcvtdq2pd %xmm0, %ymm0 -; AVX1-NEXT: vmulpd .LCPI13_1(%rip), %ymm0, %ymm0 +; AVX1-NEXT: vmulpd {{.*}}(%rip), %ymm0, %ymm0 ; AVX1-NEXT: vaddpd %ymm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; -; AVX2-LABEL: uitofp_4vf64_i32: +; AVX2-LABEL: uitofp_4i32_to_4f64: ; AVX2: # BB#0: ; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1 ; AVX2-NEXT: vcvtdq2pd %xmm1, %ymm1 -; AVX2-NEXT: vbroadcastsd .LCPI13_0(%rip), %ymm2 +; AVX2-NEXT: vbroadcastsd {{.*}}(%rip), %ymm2 ; AVX2-NEXT: vmulpd %ymm2, %ymm1, %ymm1 -; AVX2-NEXT: vpbroadcastd .LCPI13_1(%rip), %xmm2 +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %xmm2 ; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 ; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm0 ; AVX2-NEXT: vaddpd %ymm0, %ymm1, %ymm0 @@ -460,18 +677,18 @@ define <4 x double> @uitofp_4vf64_i32(<4 x i32> %a) { ret <4 x double> %cvt } -define <4 x double> @uitofp_4vf64_i16(<8 x i16> %a) { -; SSE2-LABEL: uitofp_4vf64_i16: -; SSE2: # BB#0: -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] -; SSE2-NEXT: cvtdq2pd %xmm0, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE2-NEXT: cvtdq2pd %xmm0, %xmm1 -; SSE2-NEXT: movaps %xmm2, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: uitofp_4vf64_i16: +define <4 x double> @uitofp_4i16_to_4f64(<8 x i16> %a) { +; SSE-LABEL: uitofp_4i16_to_4f64: +; SSE: # BB#0: +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE-NEXT: cvtdq2pd %xmm0, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; SSE-NEXT: cvtdq2pd %xmm0, %xmm1 +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: uitofp_4i16_to_4f64: ; AVX: # BB#0: ; AVX-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero ; AVX-NEXT: vcvtdq2pd %xmm0, %ymm0 @@ -481,19 +698,46 @@ define <4 x double> @uitofp_4vf64_i16(<8 x i16> %a) { ret <4 x double> %cvt } -define <4 x double> @uitofp_4vf64_i8(<16 x i8> %a) { -; SSE2-LABEL: uitofp_4vf64_i8: -; SSE2: # BB#0: -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] -; SSE2-NEXT: cvtdq2pd %xmm0, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE2-NEXT: cvtdq2pd %xmm0, %xmm1 -; SSE2-NEXT: movaps %xmm2, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: uitofp_4vf64_i8: +define <4 x double> @uitofp_8i16_to_4f64(<8 x i16> %a) { +; SSE-LABEL: uitofp_8i16_to_4f64: +; SSE: # BB#0: +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE-NEXT: cvtdq2pd %xmm0, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; SSE-NEXT: cvtdq2pd %xmm0, %xmm1 +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: uitofp_8i16_to_4f64: +; AVX1: # BB#0: +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX1-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: uitofp_8i16_to_4f64: +; AVX2: # BB#0: +; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX2-NEXT: retq + %cvt = uitofp <8 x i16> %a to <8 x double> + %shuf = shufflevector <8 x double> %cvt, <8 x double> undef, <4 x i32> + ret <4 x double> %shuf +} + +define <4 x double> @uitofp_4i8_to_4f64(<16 x i8> %a) { +; SSE-LABEL: uitofp_4i8_to_4f64: +; SSE: # BB#0: +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE-NEXT: cvtdq2pd %xmm0, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; SSE-NEXT: cvtdq2pd %xmm0, %xmm1 +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: uitofp_4i8_to_4f64: ; AVX: # BB#0: ; AVX-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero ; AVX-NEXT: vcvtdq2pd %xmm0, %ymm0 @@ -503,38 +747,86 @@ define <4 x double> @uitofp_4vf64_i8(<16 x i8> %a) { ret <4 x double> %cvt } +define <4 x double> @uitofp_16i8_to_4f64(<16 x i8> %a) { +; SSE-LABEL: uitofp_16i8_to_4f64: +; SSE: # BB#0: +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE-NEXT: cvtdq2pd %xmm0, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; SSE-NEXT: cvtdq2pd %xmm0, %xmm1 +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: uitofp_16i8_to_4f64: +; AVX1: # BB#0: +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX1-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: uitofp_16i8_to_4f64: +; AVX2: # BB#0: +; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX2-NEXT: vcvtdq2pd %xmm0, %ymm0 +; AVX2-NEXT: retq + %cvt = uitofp <16 x i8> %a to <16 x double> + %shuf = shufflevector <16 x double> %cvt, <16 x double> undef, <4 x i32> + ret <4 x double> %shuf +} + ; ; Signed Integer to Float ; -define <4 x float> @sitofp_4vf32(<4 x i32> %a) { -; SSE2-LABEL: sitofp_4vf32: -; SSE2: # BB#0: -; SSE2-NEXT: cvtdq2ps %xmm0, %xmm0 -; SSE2-NEXT: retq +define <4 x float> @sitofp_2i64_to_4f32(<2 x i64> %a) { +; SSE-LABEL: sitofp_2i64_to_4f32: +; SSE: # BB#0: +; SSE-NEXT: movd %xmm0, %rax +; SSE-NEXT: cvtsi2ssq %rax, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; SSE-NEXT: movd %xmm0, %rax +; SSE-NEXT: xorps %xmm0, %xmm0 +; SSE-NEXT: cvtsi2ssq %rax, %xmm0 +; SSE-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; SSE-NEXT: movaps %xmm1, %xmm0 +; SSE-NEXT: retq ; -; AVX-LABEL: sitofp_4vf32: +; AVX-LABEL: sitofp_2i64_to_4f32: ; AVX: # BB#0: -; AVX-NEXT: vcvtdq2ps %xmm0, %xmm0 +; AVX-NEXT: vpextrq $1, %xmm0, %rax +; AVX-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm1 +; AVX-NEXT: vmovq %xmm0, %rax +; AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm0 +; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3] +; AVX-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm1 +; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3] +; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] ; AVX-NEXT: retq - %cvt = sitofp <4 x i32> %a to <4 x float> - ret <4 x float> %cvt + %cvt = sitofp <2 x i64> %a to <2 x float> + %ext = shufflevector <2 x float> %cvt, <2 x float> undef, <4 x i32> + ret <4 x float> %ext } -define <4 x float> @sitofp_4vf32_i64(<2 x i64> %a) { -; SSE2-LABEL: sitofp_4vf32_i64: -; SSE2: # BB#0: -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: cvtsi2ssq %rax, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: xorps %xmm0, %xmm0 -; SSE2-NEXT: cvtsi2ssq %rax, %xmm0 -; SSE2-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSE2-NEXT: movaps %xmm1, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: sitofp_4vf32_i64: +define <4 x float> @sitofp_4i64_to_4f32_undef(<2 x i64> %a) { +; SSE-LABEL: sitofp_4i64_to_4f32_undef: +; SSE: # BB#0: +; SSE-NEXT: cvtsi2ssq %rax, %xmm2 +; SSE-NEXT: movd %xmm0, %rax +; SSE-NEXT: cvtsi2ssq %rax, %xmm1 +; SSE-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; SSE-NEXT: movd %xmm0, %rax +; SSE-NEXT: xorps %xmm0, %xmm0 +; SSE-NEXT: cvtsi2ssq %rax, %xmm0 +; SSE-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSE-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; SSE-NEXT: movaps %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: sitofp_4i64_to_4f32_undef: ; AVX: # BB#0: ; AVX-NEXT: vpextrq $1, %xmm0, %rax ; AVX-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm1 @@ -546,20 +838,34 @@ define <4 x float> @sitofp_4vf32_i64(<2 x i64> %a) { ; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3] ; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] ; AVX-NEXT: retq - %cvt = sitofp <2 x i64> %a to <2 x float> - %ext = shufflevector <2 x float> %cvt, <2 x float> undef, <4 x i32> - ret <4 x float> %ext + %ext = shufflevector <2 x i64> %a, <2 x i64> undef, <4 x i32> + %cvt = sitofp <4 x i64> %ext to <4 x float> + ret <4 x float> %cvt +} + +define <4 x float> @sitofp_4i32_to_4f32(<4 x i32> %a) { +; SSE-LABEL: sitofp_4i32_to_4f32: +; SSE: # BB#0: +; SSE-NEXT: cvtdq2ps %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: sitofp_4i32_to_4f32: +; AVX: # BB#0: +; AVX-NEXT: vcvtdq2ps %xmm0, %xmm0 +; AVX-NEXT: retq + %cvt = sitofp <4 x i32> %a to <4 x float> + ret <4 x float> %cvt } -define <4 x float> @sitofp_4vf32_i16(<8 x i16> %a) { -; SSE2-LABEL: sitofp_4vf32_i16: -; SSE2: # BB#0: -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] -; SSE2-NEXT: psrad $16, %xmm0 -; SSE2-NEXT: cvtdq2ps %xmm0, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: sitofp_4vf32_i16: +define <4 x float> @sitofp_4i16_to_4f32(<8 x i16> %a) { +; SSE-LABEL: sitofp_4i16_to_4f32: +; SSE: # BB#0: +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE-NEXT: psrad $16, %xmm0 +; SSE-NEXT: cvtdq2ps %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: sitofp_4i16_to_4f32: ; AVX: # BB#0: ; AVX-NEXT: vpmovsxwd %xmm0, %xmm0 ; AVX-NEXT: vcvtdq2ps %xmm0, %xmm0 @@ -569,16 +875,45 @@ define <4 x float> @sitofp_4vf32_i16(<8 x i16> %a) { ret <4 x float> %cvt } -define <4 x float> @sitofp_4vf32_i8(<16 x i8> %a) { -; SSE2-LABEL: sitofp_4vf32_i8: -; SSE2: # BB#0: -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] -; SSE2-NEXT: psrad $24, %xmm0 -; SSE2-NEXT: cvtdq2ps %xmm0, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: sitofp_4vf32_i8: +define <4 x float> @sitofp_8i16_to_4f32(<8 x i16> %a) { +; SSE-LABEL: sitofp_8i16_to_4f32: +; SSE: # BB#0: +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE-NEXT: psrad $16, %xmm0 +; SSE-NEXT: cvtdq2ps %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: sitofp_8i16_to_4f32: +; AVX1: # BB#0: +; AVX1-NEXT: vpmovsxwd %xmm0, %xmm1 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; AVX1-NEXT: vpmovsxwd %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: vcvtdq2ps %ymm0, %ymm0 +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: sitofp_8i16_to_4f32: +; AVX2: # BB#0: +; AVX2-NEXT: vpmovsxwd %xmm0, %ymm0 +; AVX2-NEXT: vcvtdq2ps %ymm0, %ymm0 +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq + %cvt = sitofp <8 x i16> %a to <8 x float> + %shuf = shufflevector <8 x float> %cvt, <8 x float> undef, <4 x i32> + ret <4 x float> %shuf +} + +define <4 x float> @sitofp_4i8_to_4f32(<16 x i8> %a) { +; SSE-LABEL: sitofp_4i8_to_4f32: +; SSE: # BB#0: +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE-NEXT: psrad $24, %xmm0 +; SSE-NEXT: cvtdq2ps %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: sitofp_4i8_to_4f32: ; AVX: # BB#0: ; AVX-NEXT: vpmovsxbd %xmm0, %xmm0 ; AVX-NEXT: vcvtdq2ps %xmm0, %xmm0 @@ -588,43 +923,59 @@ define <4 x float> @sitofp_4vf32_i8(<16 x i8> %a) { ret <4 x float> %cvt } -define <8 x float> @sitofp_8vf32(<8 x i32> %a) { -; SSE2-LABEL: sitofp_8vf32: -; SSE2: # BB#0: -; SSE2-NEXT: cvtdq2ps %xmm0, %xmm0 -; SSE2-NEXT: cvtdq2ps %xmm1, %xmm1 -; SSE2-NEXT: retq -; -; AVX-LABEL: sitofp_8vf32: -; AVX: # BB#0: -; AVX-NEXT: vcvtdq2ps %ymm0, %ymm0 -; AVX-NEXT: retq - %cvt = sitofp <8 x i32> %a to <8 x float> - ret <8 x float> %cvt +define <4 x float> @sitofp_16i8_to_4f32(<16 x i8> %a) { +; SSE-LABEL: sitofp_16i8_to_4f32: +; SSE: # BB#0: +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE-NEXT: psrad $24, %xmm0 +; SSE-NEXT: cvtdq2ps %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: sitofp_16i8_to_4f32: +; AVX1: # BB#0: +; AVX1-NEXT: vpmovsxbd %xmm0, %xmm1 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; AVX1-NEXT: vpmovsxbd %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: vcvtdq2ps %ymm0, %ymm0 +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: sitofp_16i8_to_4f32: +; AVX2: # BB#0: +; AVX2-NEXT: vpmovsxbw %xmm0, %ymm0 +; AVX2-NEXT: vpmovsxwd %xmm0, %ymm0 +; AVX2-NEXT: vcvtdq2ps %ymm0, %ymm0 +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq + %cvt = sitofp <16 x i8> %a to <16 x float> + %shuf = shufflevector <16 x float> %cvt, <16 x float> undef, <4 x i32> + ret <4 x float> %shuf } -define <4 x float> @sitofp_4vf32_4i64(<4 x i64> %a) { -; SSE2-LABEL: sitofp_4vf32_4i64: -; SSE2: # BB#0: -; SSE2-NEXT: movd %xmm1, %rax -; SSE2-NEXT: cvtsi2ssq %rax, %xmm3 -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: cvtsi2ssq %rax, %xmm2 -; SSE2-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] -; SSE2-NEXT: movd %xmm1, %rax -; SSE2-NEXT: xorps %xmm1, %xmm1 -; SSE2-NEXT: cvtsi2ssq %rax, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: xorps %xmm0, %xmm0 -; SSE2-NEXT: cvtsi2ssq %rax, %xmm0 -; SSE2-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] -; SSE2-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] -; SSE2-NEXT: movaps %xmm2, %xmm0 -; SSE2-NEXT: retq -; -; AVX1-LABEL: sitofp_4vf32_4i64: +define <4 x float> @sitofp_4i64_to_4f32(<4 x i64> %a) { +; SSE-LABEL: sitofp_4i64_to_4f32: +; SSE: # BB#0: +; SSE-NEXT: movd %xmm1, %rax +; SSE-NEXT: cvtsi2ssq %rax, %xmm3 +; SSE-NEXT: movd %xmm0, %rax +; SSE-NEXT: cvtsi2ssq %rax, %xmm2 +; SSE-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] +; SSE-NEXT: movd %xmm1, %rax +; SSE-NEXT: xorps %xmm1, %xmm1 +; SSE-NEXT: cvtsi2ssq %rax, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; SSE-NEXT: movd %xmm0, %rax +; SSE-NEXT: xorps %xmm0, %xmm0 +; SSE-NEXT: cvtsi2ssq %rax, %xmm0 +; SSE-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: sitofp_4i64_to_4f32: ; AVX1: # BB#0: ; AVX1-NEXT: vpextrq $1, %xmm0, %rax ; AVX1-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm1 @@ -642,7 +993,7 @@ define <4 x float> @sitofp_4vf32_4i64(<4 x i64> %a) { ; AVX1-NEXT: vzeroupper ; AVX1-NEXT: retq ; -; AVX2-LABEL: sitofp_4vf32_4i64: +; AVX2-LABEL: sitofp_4i64_to_4f32: ; AVX2: # BB#0: ; AVX2-NEXT: vpextrq $1, %xmm0, %rax ; AVX2-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm1 @@ -663,20 +1014,34 @@ define <4 x float> @sitofp_4vf32_4i64(<4 x i64> %a) { ret <4 x float> %cvt } -define <8 x float> @sitofp_8vf32_i16(<8 x i16> %a) { -; SSE2-LABEL: sitofp_8vf32_i16: -; SSE2: # BB#0: -; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] -; SSE2-NEXT: psrad $16, %xmm1 -; SSE2-NEXT: cvtdq2ps %xmm1, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] -; SSE2-NEXT: psrad $16, %xmm0 -; SSE2-NEXT: cvtdq2ps %xmm0, %xmm1 -; SSE2-NEXT: movaps %xmm2, %xmm0 -; SSE2-NEXT: retq -; -; AVX1-LABEL: sitofp_8vf32_i16: +define <8 x float> @sitofp_8i32_to_8f32(<8 x i32> %a) { +; SSE-LABEL: sitofp_8i32_to_8f32: +; SSE: # BB#0: +; SSE-NEXT: cvtdq2ps %xmm0, %xmm0 +; SSE-NEXT: cvtdq2ps %xmm1, %xmm1 +; SSE-NEXT: retq +; +; AVX-LABEL: sitofp_8i32_to_8f32: +; AVX: # BB#0: +; AVX-NEXT: vcvtdq2ps %ymm0, %ymm0 +; AVX-NEXT: retq + %cvt = sitofp <8 x i32> %a to <8 x float> + ret <8 x float> %cvt +} + +define <8 x float> @sitofp_8i16_to_8f32(<8 x i16> %a) { +; SSE-LABEL: sitofp_8i16_to_8f32: +; SSE: # BB#0: +; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSE-NEXT: psrad $16, %xmm1 +; SSE-NEXT: cvtdq2ps %xmm1, %xmm2 +; SSE-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7] +; SSE-NEXT: psrad $16, %xmm0 +; SSE-NEXT: cvtdq2ps %xmm0, %xmm1 +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: sitofp_8i16_to_8f32: ; AVX1: # BB#0: ; AVX1-NEXT: vpmovsxwd %xmm0, %xmm1 ; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] @@ -685,7 +1050,7 @@ define <8 x float> @sitofp_8vf32_i16(<8 x i16> %a) { ; AVX1-NEXT: vcvtdq2ps %ymm0, %ymm0 ; AVX1-NEXT: retq ; -; AVX2-LABEL: sitofp_8vf32_i16: +; AVX2-LABEL: sitofp_8i16_to_8f32: ; AVX2: # BB#0: ; AVX2-NEXT: vpmovsxwd %xmm0, %ymm0 ; AVX2-NEXT: vcvtdq2ps %ymm0, %ymm0 @@ -694,22 +1059,22 @@ define <8 x float> @sitofp_8vf32_i16(<8 x i16> %a) { ret <8 x float> %cvt } -define <8 x float> @sitofp_8vf32_i8(<16 x i8> %a) { -; SSE2-LABEL: sitofp_8vf32_i8: -; SSE2: # BB#0: -; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] -; SSE2-NEXT: psrad $24, %xmm1 -; SSE2-NEXT: cvtdq2ps %xmm1, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] -; SSE2-NEXT: psrad $24, %xmm0 -; SSE2-NEXT: cvtdq2ps %xmm0, %xmm1 -; SSE2-NEXT: movaps %xmm2, %xmm0 -; SSE2-NEXT: retq -; -; AVX1-LABEL: sitofp_8vf32_i8: +define <8 x float> @sitofp_8i8_to_8f32(<16 x i8> %a) { +; SSE-LABEL: sitofp_8i8_to_8f32: +; SSE: # BB#0: +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] +; SSE-NEXT: psrad $24, %xmm1 +; SSE-NEXT: cvtdq2ps %xmm1, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE-NEXT: psrad $24, %xmm0 +; SSE-NEXT: cvtdq2ps %xmm0, %xmm1 +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: sitofp_8i8_to_8f32: ; AVX1: # BB#0: ; AVX1-NEXT: vpmovsxbd %xmm0, %xmm1 ; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] @@ -718,9 +1083,9 @@ define <8 x float> @sitofp_8vf32_i8(<16 x i8> %a) { ; AVX1-NEXT: vcvtdq2ps %ymm0, %ymm0 ; AVX1-NEXT: retq ; -; AVX2-LABEL: sitofp_8vf32_i8: +; AVX2-LABEL: sitofp_8i8_to_8f32: ; AVX2: # BB#0: -; AVX2-NEXT: vpmovzxbd %xmm0, %ymm0 +; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero ; AVX2-NEXT: vpslld $24, %ymm0, %ymm0 ; AVX2-NEXT: vpsrad $24, %ymm0, %ymm0 ; AVX2-NEXT: vcvtdq2ps %ymm0, %ymm0 @@ -730,125 +1095,124 @@ define <8 x float> @sitofp_8vf32_i8(<16 x i8> %a) { ret <8 x float> %cvt } +define <8 x float> @sitofp_16i8_to_8f32(<16 x i8> %a) { +; SSE-LABEL: sitofp_16i8_to_8f32: +; SSE: # BB#0: +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] +; SSE-NEXT: psrad $24, %xmm1 +; SSE-NEXT: cvtdq2ps %xmm1, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE-NEXT: psrad $24, %xmm0 +; SSE-NEXT: cvtdq2ps %xmm0, %xmm1 +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq ; -; Unsigned Integer to Float -; - -define <4 x float> @uitofp_4vf32(<4 x i32> %a) { -; SSE2-LABEL: uitofp_4vf32: -; SSE2: # BB#0: -; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535] -; SSE2-NEXT: pand %xmm0, %xmm1 -; SSE2-NEXT: por .LCPI24_1(%rip), %xmm1 -; SSE2-NEXT: psrld $16, %xmm0 -; SSE2-NEXT: por .LCPI24_2(%rip), %xmm0 -; SSE2-NEXT: addps .LCPI24_3(%rip), %xmm0 -; SSE2-NEXT: addps %xmm1, %xmm0 -; SSE2-NEXT: retq -; -; AVX1-LABEL: uitofp_4vf32: +; AVX1-LABEL: sitofp_16i8_to_8f32: ; AVX1: # BB#0: -; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7] -; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0 -; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7] -; AVX1-NEXT: vaddps .LCPI24_2(%rip), %xmm0, %xmm0 -; AVX1-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vpmovsxbd %xmm0, %xmm1 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; AVX1-NEXT: vpmovsxbd %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: vcvtdq2ps %ymm0, %ymm0 ; AVX1-NEXT: retq ; -; AVX2-LABEL: uitofp_4vf32: +; AVX2-LABEL: sitofp_16i8_to_8f32: ; AVX2: # BB#0: -; AVX2-NEXT: vpbroadcastd .LCPI24_0(%rip), %xmm1 -; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] -; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0 -; AVX2-NEXT: vpbroadcastd .LCPI24_1(%rip), %xmm2 -; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7] -; AVX2-NEXT: vbroadcastss .LCPI24_2(%rip), %xmm2 -; AVX2-NEXT: vaddps %xmm2, %xmm0, %xmm0 -; AVX2-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vpmovsxbw %xmm0, %ymm0 +; AVX2-NEXT: vpmovsxwd %xmm0, %ymm0 +; AVX2-NEXT: vcvtdq2ps %ymm0, %ymm0 ; AVX2-NEXT: retq - %cvt = uitofp <4 x i32> %a to <4 x float> - ret <4 x float> %cvt + %cvt = sitofp <16 x i8> %a to <16 x float> + %shuf = shufflevector <16 x float> %cvt, <16 x float> undef, <8 x i32> + ret <8 x float> %shuf } -define <4 x float> @uitofp_4vf32_i64(<2 x i64> %a) { -; SSE2-LABEL: uitofp_4vf32_i64: -; SSE2: # BB#0: -; SSE2-NEXT: movdqa %xmm0, %xmm1 -; SSE2-NEXT: movd %xmm1, %rax -; SSE2-NEXT: movl %eax, %ecx -; SSE2-NEXT: andl $1, %ecx -; SSE2-NEXT: testq %rax, %rax -; SSE2-NEXT: js .LBB25_1 -; SSE2-NEXT: # BB#2: -; SSE2-NEXT: xorps %xmm0, %xmm0 -; SSE2-NEXT: cvtsi2ssq %rax, %xmm0 -; SSE2-NEXT: jmp .LBB25_3 -; SSE2-NEXT: .LBB25_1: -; SSE2-NEXT: shrq %rax -; SSE2-NEXT: orq %rax, %rcx -; SSE2-NEXT: xorps %xmm0, %xmm0 -; SSE2-NEXT: cvtsi2ssq %rcx, %xmm0 -; SSE2-NEXT: addss %xmm0, %xmm0 -; SSE2-NEXT: .LBB25_3: -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] -; SSE2-NEXT: movd %xmm1, %rax -; SSE2-NEXT: movl %eax, %ecx -; SSE2-NEXT: andl $1, %ecx -; SSE2-NEXT: testq %rax, %rax -; SSE2-NEXT: js .LBB25_4 -; SSE2-NEXT: # BB#5: -; SSE2-NEXT: xorps %xmm1, %xmm1 -; SSE2-NEXT: cvtsi2ssq %rax, %xmm1 -; SSE2-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] -; SSE2-NEXT: retq -; SSE2-NEXT: .LBB25_4: -; SSE2-NEXT: shrq %rax -; SSE2-NEXT: orq %rax, %rcx -; SSE2-NEXT: xorps %xmm1, %xmm1 -; SSE2-NEXT: cvtsi2ssq %rcx, %xmm1 -; SSE2-NEXT: addss %xmm1, %xmm1 -; SSE2-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] -; SSE2-NEXT: retq -; -; AVX-LABEL: uitofp_4vf32_i64: +; +; Unsigned Integer to Float +; + +define <4 x float> @uitofp_2i64_to_4f32(<2 x i64> %a) { +; SSE-LABEL: uitofp_2i64_to_4f32: +; SSE: # BB#0: +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: movd %xmm1, %rax +; SSE-NEXT: movl %eax, %ecx +; SSE-NEXT: andl $1, %ecx +; SSE-NEXT: testq %rax, %rax +; SSE-NEXT: js .LBB38_1 +; SSE-NEXT: # BB#2: +; SSE-NEXT: xorps %xmm0, %xmm0 +; SSE-NEXT: cvtsi2ssq %rax, %xmm0 +; SSE-NEXT: jmp .LBB38_3 +; SSE-NEXT: .LBB38_1: +; SSE-NEXT: shrq %rax +; SSE-NEXT: orq %rax, %rcx +; SSE-NEXT: xorps %xmm0, %xmm0 +; SSE-NEXT: cvtsi2ssq %rcx, %xmm0 +; SSE-NEXT: addss %xmm0, %xmm0 +; SSE-NEXT: .LBB38_3: +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] +; SSE-NEXT: movd %xmm1, %rax +; SSE-NEXT: movl %eax, %ecx +; SSE-NEXT: andl $1, %ecx +; SSE-NEXT: testq %rax, %rax +; SSE-NEXT: js .LBB38_4 +; SSE-NEXT: # BB#5: +; SSE-NEXT: xorps %xmm1, %xmm1 +; SSE-NEXT: cvtsi2ssq %rax, %xmm1 +; SSE-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE-NEXT: retq +; SSE-NEXT: .LBB38_4: +; SSE-NEXT: shrq %rax +; SSE-NEXT: orq %rax, %rcx +; SSE-NEXT: xorps %xmm1, %xmm1 +; SSE-NEXT: cvtsi2ssq %rcx, %xmm1 +; SSE-NEXT: addss %xmm1, %xmm1 +; SSE-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE-NEXT: retq +; +; AVX-LABEL: uitofp_2i64_to_4f32: ; AVX: # BB#0: ; AVX-NEXT: vpextrq $1, %xmm0, %rax ; AVX-NEXT: movl %eax, %ecx ; AVX-NEXT: andl $1, %ecx ; AVX-NEXT: testq %rax, %rax -; AVX-NEXT: js .LBB25_1 +; AVX-NEXT: js .LBB38_1 ; AVX-NEXT: # BB#2: ; AVX-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm1 -; AVX-NEXT: jmp .LBB25_3 -; AVX-NEXT: .LBB25_1: +; AVX-NEXT: jmp .LBB38_3 +; AVX-NEXT: .LBB38_1: ; AVX-NEXT: shrq %rax ; AVX-NEXT: orq %rax, %rcx ; AVX-NEXT: vcvtsi2ssq %rcx, %xmm0, %xmm1 ; AVX-NEXT: vaddss %xmm1, %xmm1, %xmm1 -; AVX-NEXT: .LBB25_3: +; AVX-NEXT: .LBB38_3: ; AVX-NEXT: vmovq %xmm0, %rax ; AVX-NEXT: movl %eax, %ecx ; AVX-NEXT: andl $1, %ecx ; AVX-NEXT: testq %rax, %rax -; AVX-NEXT: js .LBB25_4 +; AVX-NEXT: js .LBB38_4 ; AVX-NEXT: # BB#5: ; AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0 ; AVX-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm0 -; AVX-NEXT: jmp .LBB25_6 -; AVX-NEXT: .LBB25_4: +; AVX-NEXT: jmp .LBB38_6 +; AVX-NEXT: .LBB38_4: ; AVX-NEXT: shrq %rax ; AVX-NEXT: orq %rax, %rcx ; AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0 ; AVX-NEXT: vcvtsi2ssq %rcx, %xmm0, %xmm0 ; AVX-NEXT: vaddss %xmm0, %xmm0, %xmm0 -; AVX-NEXT: .LBB25_6: +; AVX-NEXT: .LBB38_6: ; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3] ; AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1 ; AVX-NEXT: testq %rax, %rax -; AVX-NEXT: js .LBB25_8 +; AVX-NEXT: js .LBB38_8 ; AVX-NEXT: # BB#7: ; AVX-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm1 -; AVX-NEXT: .LBB25_8: +; AVX-NEXT: .LBB38_8: ; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3] ; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] ; AVX-NEXT: retq @@ -857,15 +1221,147 @@ define <4 x float> @uitofp_4vf32_i64(<2 x i64> %a) { ret <4 x float> %ext } -define <4 x float> @uitofp_4vf32_i16(<8 x i16> %a) { -; SSE2-LABEL: uitofp_4vf32_i16: -; SSE2: # BB#0: -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] -; SSE2-NEXT: cvtdq2ps %xmm0, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: uitofp_4vf32_i16: +define <4 x float> @uitofp_4i64_to_4f32_undef(<2 x i64> %a) { +; SSE-LABEL: uitofp_4i64_to_4f32_undef: +; SSE: # BB#0: +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: testq %rax, %rax +; SSE-NEXT: xorps %xmm2, %xmm2 +; SSE-NEXT: js .LBB39_2 +; SSE-NEXT: # BB#1: +; SSE-NEXT: xorps %xmm2, %xmm2 +; SSE-NEXT: cvtsi2ssq %rax, %xmm2 +; SSE-NEXT: .LBB39_2: +; SSE-NEXT: movd %xmm1, %rax +; SSE-NEXT: movl %eax, %ecx +; SSE-NEXT: andl $1, %ecx +; SSE-NEXT: testq %rax, %rax +; SSE-NEXT: js .LBB39_3 +; SSE-NEXT: # BB#4: +; SSE-NEXT: xorps %xmm0, %xmm0 +; SSE-NEXT: cvtsi2ssq %rax, %xmm0 +; SSE-NEXT: jmp .LBB39_5 +; SSE-NEXT: .LBB39_3: +; SSE-NEXT: shrq %rax +; SSE-NEXT: orq %rax, %rcx +; SSE-NEXT: xorps %xmm0, %xmm0 +; SSE-NEXT: cvtsi2ssq %rcx, %xmm0 +; SSE-NEXT: addss %xmm0, %xmm0 +; SSE-NEXT: .LBB39_5: +; SSE-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] +; SSE-NEXT: movd %xmm1, %rax +; SSE-NEXT: movl %eax, %ecx +; SSE-NEXT: andl $1, %ecx +; SSE-NEXT: testq %rax, %rax +; SSE-NEXT: js .LBB39_6 +; SSE-NEXT: # BB#7: +; SSE-NEXT: xorps %xmm1, %xmm1 +; SSE-NEXT: cvtsi2ssq %rax, %xmm1 +; SSE-NEXT: jmp .LBB39_8 +; SSE-NEXT: .LBB39_6: +; SSE-NEXT: shrq %rax +; SSE-NEXT: orq %rax, %rcx +; SSE-NEXT: xorps %xmm1, %xmm1 +; SSE-NEXT: cvtsi2ssq %rcx, %xmm1 +; SSE-NEXT: addss %xmm1, %xmm1 +; SSE-NEXT: .LBB39_8: +; SSE-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; SSE-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE-NEXT: retq +; +; AVX-LABEL: uitofp_4i64_to_4f32_undef: +; AVX: # BB#0: +; AVX-NEXT: vpextrq $1, %xmm0, %rax +; AVX-NEXT: movl %eax, %ecx +; AVX-NEXT: andl $1, %ecx +; AVX-NEXT: testq %rax, %rax +; AVX-NEXT: js .LBB39_1 +; AVX-NEXT: # BB#2: +; AVX-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm1 +; AVX-NEXT: jmp .LBB39_3 +; AVX-NEXT: .LBB39_1: +; AVX-NEXT: shrq %rax +; AVX-NEXT: orq %rax, %rcx +; AVX-NEXT: vcvtsi2ssq %rcx, %xmm0, %xmm1 +; AVX-NEXT: vaddss %xmm1, %xmm1, %xmm1 +; AVX-NEXT: .LBB39_3: +; AVX-NEXT: vmovq %xmm0, %rax +; AVX-NEXT: movl %eax, %ecx +; AVX-NEXT: andl $1, %ecx +; AVX-NEXT: testq %rax, %rax +; AVX-NEXT: js .LBB39_4 +; AVX-NEXT: # BB#5: +; AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm0 +; AVX-NEXT: jmp .LBB39_6 +; AVX-NEXT: .LBB39_4: +; AVX-NEXT: shrq %rax +; AVX-NEXT: orq %rax, %rcx +; AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vcvtsi2ssq %rcx, %xmm0, %xmm0 +; AVX-NEXT: vaddss %xmm0, %xmm0, %xmm0 +; AVX-NEXT: .LBB39_6: +; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3] +; AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1 +; AVX-NEXT: testq %rax, %rax +; AVX-NEXT: js .LBB39_8 +; AVX-NEXT: # BB#7: +; AVX-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm1 +; AVX-NEXT: .LBB39_8: +; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3] +; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] +; AVX-NEXT: retq + %ext = shufflevector <2 x i64> %a, <2 x i64> undef, <4 x i32> + %cvt = uitofp <4 x i64> %ext to <4 x float> + ret <4 x float> %cvt +} + +define <4 x float> @uitofp_4i32_to_4f32(<4 x i32> %a) { +; SSE-LABEL: uitofp_4i32_to_4f32: +; SSE: # BB#0: +; SSE-NEXT: movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535] +; SSE-NEXT: pand %xmm0, %xmm1 +; SSE-NEXT: por {{.*}}(%rip), %xmm1 +; SSE-NEXT: psrld $16, %xmm0 +; SSE-NEXT: por {{.*}}(%rip), %xmm0 +; SSE-NEXT: addps {{.*}}(%rip), %xmm0 +; SSE-NEXT: addps %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: uitofp_4i32_to_4f32: +; AVX1: # BB#0: +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7] +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7] +; AVX1-NEXT: vaddps {{.*}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: uitofp_4i32_to_4f32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %xmm1 +; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] +; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %xmm2 +; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7] +; AVX2-NEXT: vbroadcastss {{.*}}(%rip), %xmm2 +; AVX2-NEXT: vaddps %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: retq + %cvt = uitofp <4 x i32> %a to <4 x float> + ret <4 x float> %cvt +} + +define <4 x float> @uitofp_4i16_to_4f32(<8 x i16> %a) { +; SSE-LABEL: uitofp_4i16_to_4f32: +; SSE: # BB#0: +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE-NEXT: cvtdq2ps %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: uitofp_4i16_to_4f32: ; AVX: # BB#0: ; AVX-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero ; AVX-NEXT: vcvtdq2ps %xmm0, %xmm0 @@ -875,16 +1371,45 @@ define <4 x float> @uitofp_4vf32_i16(<8 x i16> %a) { ret <4 x float> %cvt } -define <4 x float> @uitofp_4vf32_i8(<16 x i8> %a) { -; SSE2-LABEL: uitofp_4vf32_i8: -; SSE2: # BB#0: -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] -; SSE2-NEXT: cvtdq2ps %xmm0, %xmm0 -; SSE2-NEXT: retq -; -; AVX-LABEL: uitofp_4vf32_i8: +define <4 x float> @uitofp_8i16_to_4f32(<8 x i16> %a) { +; SSE-LABEL: uitofp_8i16_to_4f32: +; SSE: # BB#0: +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE-NEXT: cvtdq2ps %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: uitofp_8i16_to_4f32: +; AVX1: # BB#0: +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vcvtdq2ps %ymm0, %ymm0 +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: uitofp_8i16_to_4f32: +; AVX2: # BB#0: +; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX2-NEXT: vcvtdq2ps %ymm0, %ymm0 +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq + %cvt = uitofp <8 x i16> %a to <8 x float> + %shuf = shufflevector <8 x float> %cvt, <8 x float> undef, <4 x i32> + ret <4 x float> %shuf +} + +define <4 x float> @uitofp_4i8_to_4f32(<16 x i8> %a) { +; SSE-LABEL: uitofp_4i8_to_4f32: +; SSE: # BB#0: +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE-NEXT: cvtdq2ps %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: uitofp_4i8_to_4f32: ; AVX: # BB#0: ; AVX-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero ; AVX-NEXT: vcvtdq2ps %xmm0, %xmm0 @@ -894,186 +1419,168 @@ define <4 x float> @uitofp_4vf32_i8(<16 x i8> %a) { ret <4 x float> %cvt } -define <8 x float> @uitofp_8vf32(<8 x i32> %a) { -; SSE2-LABEL: uitofp_8vf32: -; SSE2: # BB#0: -; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535] -; SSE2-NEXT: movdqa %xmm0, %xmm3 -; SSE2-NEXT: pand %xmm2, %xmm3 -; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1258291200,1258291200,1258291200,1258291200] -; SSE2-NEXT: por %xmm4, %xmm3 -; SSE2-NEXT: psrld $16, %xmm0 -; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [1392508928,1392508928,1392508928,1392508928] -; SSE2-NEXT: por %xmm5, %xmm0 -; SSE2-NEXT: movaps {{.*#+}} xmm6 = [-5.497642e+11,-5.497642e+11,-5.497642e+11,-5.497642e+11] -; SSE2-NEXT: addps %xmm6, %xmm0 -; SSE2-NEXT: addps %xmm3, %xmm0 -; SSE2-NEXT: pand %xmm1, %xmm2 -; SSE2-NEXT: por %xmm4, %xmm2 -; SSE2-NEXT: psrld $16, %xmm1 -; SSE2-NEXT: por %xmm5, %xmm1 -; SSE2-NEXT: addps %xmm6, %xmm1 -; SSE2-NEXT: addps %xmm2, %xmm1 -; SSE2-NEXT: retq -; -; AVX1-LABEL: uitofp_8vf32: +define <4 x float> @uitofp_16i8_to_4f32(<16 x i8> %a) { +; SSE-LABEL: uitofp_16i8_to_4f32: +; SSE: # BB#0: +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE-NEXT: cvtdq2ps %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: uitofp_16i8_to_4f32: ; AVX1: # BB#0: -; AVX1-NEXT: vandps .LCPI28_0(%rip), %ymm0, %ymm1 -; AVX1-NEXT: vcvtdq2ps %ymm1, %ymm1 -; AVX1-NEXT: vpsrld $16, %xmm0, %xmm2 -; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 -; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0 -; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0 +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: vcvtdq2ps %ymm0, %ymm0 -; AVX1-NEXT: vmulps .LCPI28_1(%rip), %ymm0, %ymm0 -; AVX1-NEXT: vaddps %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vzeroupper ; AVX1-NEXT: retq ; -; AVX2-LABEL: uitofp_8vf32: +; AVX2-LABEL: uitofp_16i8_to_4f32: ; AVX2: # BB#0: -; AVX2-NEXT: vpbroadcastd .LCPI28_0(%rip), %ymm1 -; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15] -; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0 -; AVX2-NEXT: vpbroadcastd .LCPI28_1(%rip), %ymm2 -; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7],ymm0[8],ymm2[9],ymm0[10],ymm2[11],ymm0[12],ymm2[13],ymm0[14],ymm2[15] -; AVX2-NEXT: vbroadcastss .LCPI28_2(%rip), %ymm2 -; AVX2-NEXT: vaddps %ymm2, %ymm0, %ymm0 -; AVX2-NEXT: vaddps %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX2-NEXT: vcvtdq2ps %ymm0, %ymm0 +; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq - %cvt = uitofp <8 x i32> %a to <8 x float> - ret <8 x float> %cvt + %cvt = uitofp <16 x i8> %a to <16 x float> + %shuf = shufflevector <16 x float> %cvt, <16 x float> undef, <4 x i32> + ret <4 x float> %shuf } -define <4 x float> @uitofp_4vf32_4i64(<4 x i64> %a) { -; SSE2-LABEL: uitofp_4vf32_4i64: -; SSE2: # BB#0: -; SSE2-NEXT: movd %xmm1, %rax -; SSE2-NEXT: movl %eax, %ecx -; SSE2-NEXT: andl $1, %ecx -; SSE2-NEXT: testq %rax, %rax -; SSE2-NEXT: js .LBB29_1 -; SSE2-NEXT: # BB#2: -; SSE2-NEXT: cvtsi2ssq %rax, %xmm3 -; SSE2-NEXT: jmp .LBB29_3 -; SSE2-NEXT: .LBB29_1: -; SSE2-NEXT: shrq %rax -; SSE2-NEXT: orq %rax, %rcx -; SSE2-NEXT: cvtsi2ssq %rcx, %xmm3 -; SSE2-NEXT: addss %xmm3, %xmm3 -; SSE2-NEXT: .LBB29_3: -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: movl %eax, %ecx -; SSE2-NEXT: andl $1, %ecx -; SSE2-NEXT: testq %rax, %rax -; SSE2-NEXT: js .LBB29_4 -; SSE2-NEXT: # BB#5: -; SSE2-NEXT: cvtsi2ssq %rax, %xmm2 -; SSE2-NEXT: jmp .LBB29_6 -; SSE2-NEXT: .LBB29_4: -; SSE2-NEXT: shrq %rax -; SSE2-NEXT: orq %rax, %rcx -; SSE2-NEXT: cvtsi2ssq %rcx, %xmm2 -; SSE2-NEXT: addss %xmm2, %xmm2 -; SSE2-NEXT: .LBB29_6: -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] -; SSE2-NEXT: movd %xmm1, %rax -; SSE2-NEXT: movl %eax, %ecx -; SSE2-NEXT: andl $1, %ecx -; SSE2-NEXT: testq %rax, %rax -; SSE2-NEXT: js .LBB29_7 -; SSE2-NEXT: # BB#8: -; SSE2-NEXT: xorps %xmm1, %xmm1 -; SSE2-NEXT: cvtsi2ssq %rax, %xmm1 -; SSE2-NEXT: jmp .LBB29_9 -; SSE2-NEXT: .LBB29_7: -; SSE2-NEXT: shrq %rax -; SSE2-NEXT: orq %rax, %rcx -; SSE2-NEXT: xorps %xmm1, %xmm1 -; SSE2-NEXT: cvtsi2ssq %rcx, %xmm1 -; SSE2-NEXT: addss %xmm1, %xmm1 -; SSE2-NEXT: .LBB29_9: -; SSE2-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: movl %eax, %ecx -; SSE2-NEXT: andl $1, %ecx -; SSE2-NEXT: testq %rax, %rax -; SSE2-NEXT: js .LBB29_10 -; SSE2-NEXT: # BB#11: -; SSE2-NEXT: xorps %xmm0, %xmm0 -; SSE2-NEXT: cvtsi2ssq %rax, %xmm0 -; SSE2-NEXT: jmp .LBB29_12 -; SSE2-NEXT: .LBB29_10: -; SSE2-NEXT: shrq %rax -; SSE2-NEXT: orq %rax, %rcx -; SSE2-NEXT: xorps %xmm0, %xmm0 -; SSE2-NEXT: cvtsi2ssq %rcx, %xmm0 -; SSE2-NEXT: addss %xmm0, %xmm0 -; SSE2-NEXT: .LBB29_12: -; SSE2-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] -; SSE2-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] -; SSE2-NEXT: movaps %xmm2, %xmm0 -; SSE2-NEXT: retq -; -; AVX1-LABEL: uitofp_4vf32_4i64: +define <4 x float> @uitofp_4i64_to_4f32(<4 x i64> %a) { +; SSE-LABEL: uitofp_4i64_to_4f32: +; SSE: # BB#0: +; SSE-NEXT: movd %xmm1, %rax +; SSE-NEXT: movl %eax, %ecx +; SSE-NEXT: andl $1, %ecx +; SSE-NEXT: testq %rax, %rax +; SSE-NEXT: js .LBB45_1 +; SSE-NEXT: # BB#2: +; SSE-NEXT: cvtsi2ssq %rax, %xmm3 +; SSE-NEXT: jmp .LBB45_3 +; SSE-NEXT: .LBB45_1: +; SSE-NEXT: shrq %rax +; SSE-NEXT: orq %rax, %rcx +; SSE-NEXT: cvtsi2ssq %rcx, %xmm3 +; SSE-NEXT: addss %xmm3, %xmm3 +; SSE-NEXT: .LBB45_3: +; SSE-NEXT: movd %xmm0, %rax +; SSE-NEXT: movl %eax, %ecx +; SSE-NEXT: andl $1, %ecx +; SSE-NEXT: testq %rax, %rax +; SSE-NEXT: js .LBB45_4 +; SSE-NEXT: # BB#5: +; SSE-NEXT: cvtsi2ssq %rax, %xmm2 +; SSE-NEXT: jmp .LBB45_6 +; SSE-NEXT: .LBB45_4: +; SSE-NEXT: shrq %rax +; SSE-NEXT: orq %rax, %rcx +; SSE-NEXT: cvtsi2ssq %rcx, %xmm2 +; SSE-NEXT: addss %xmm2, %xmm2 +; SSE-NEXT: .LBB45_6: +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] +; SSE-NEXT: movd %xmm1, %rax +; SSE-NEXT: movl %eax, %ecx +; SSE-NEXT: andl $1, %ecx +; SSE-NEXT: testq %rax, %rax +; SSE-NEXT: js .LBB45_7 +; SSE-NEXT: # BB#8: +; SSE-NEXT: xorps %xmm1, %xmm1 +; SSE-NEXT: cvtsi2ssq %rax, %xmm1 +; SSE-NEXT: jmp .LBB45_9 +; SSE-NEXT: .LBB45_7: +; SSE-NEXT: shrq %rax +; SSE-NEXT: orq %rax, %rcx +; SSE-NEXT: xorps %xmm1, %xmm1 +; SSE-NEXT: cvtsi2ssq %rcx, %xmm1 +; SSE-NEXT: addss %xmm1, %xmm1 +; SSE-NEXT: .LBB45_9: +; SSE-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; SSE-NEXT: movd %xmm0, %rax +; SSE-NEXT: movl %eax, %ecx +; SSE-NEXT: andl $1, %ecx +; SSE-NEXT: testq %rax, %rax +; SSE-NEXT: js .LBB45_10 +; SSE-NEXT: # BB#11: +; SSE-NEXT: xorps %xmm0, %xmm0 +; SSE-NEXT: cvtsi2ssq %rax, %xmm0 +; SSE-NEXT: jmp .LBB45_12 +; SSE-NEXT: .LBB45_10: +; SSE-NEXT: shrq %rax +; SSE-NEXT: orq %rax, %rcx +; SSE-NEXT: xorps %xmm0, %xmm0 +; SSE-NEXT: cvtsi2ssq %rcx, %xmm0 +; SSE-NEXT: addss %xmm0, %xmm0 +; SSE-NEXT: .LBB45_12: +; SSE-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: uitofp_4i64_to_4f32: ; AVX1: # BB#0: ; AVX1-NEXT: vpextrq $1, %xmm0, %rax ; AVX1-NEXT: movl %eax, %ecx ; AVX1-NEXT: andl $1, %ecx ; AVX1-NEXT: testq %rax, %rax -; AVX1-NEXT: js .LBB29_1 +; AVX1-NEXT: js .LBB45_1 ; AVX1-NEXT: # BB#2: ; AVX1-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm1 -; AVX1-NEXT: jmp .LBB29_3 -; AVX1-NEXT: .LBB29_1: +; AVX1-NEXT: jmp .LBB45_3 +; AVX1-NEXT: .LBB45_1: ; AVX1-NEXT: shrq %rax ; AVX1-NEXT: orq %rax, %rcx ; AVX1-NEXT: vcvtsi2ssq %rcx, %xmm0, %xmm1 ; AVX1-NEXT: vaddss %xmm1, %xmm1, %xmm1 -; AVX1-NEXT: .LBB29_3: +; AVX1-NEXT: .LBB45_3: ; AVX1-NEXT: vmovq %xmm0, %rax ; AVX1-NEXT: movl %eax, %ecx ; AVX1-NEXT: andl $1, %ecx ; AVX1-NEXT: testq %rax, %rax -; AVX1-NEXT: js .LBB29_4 +; AVX1-NEXT: js .LBB45_4 ; AVX1-NEXT: # BB#5: ; AVX1-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm2 -; AVX1-NEXT: jmp .LBB29_6 -; AVX1-NEXT: .LBB29_4: +; AVX1-NEXT: jmp .LBB45_6 +; AVX1-NEXT: .LBB45_4: ; AVX1-NEXT: shrq %rax ; AVX1-NEXT: orq %rax, %rcx ; AVX1-NEXT: vcvtsi2ssq %rcx, %xmm0, %xmm2 ; AVX1-NEXT: vaddss %xmm2, %xmm2, %xmm2 -; AVX1-NEXT: .LBB29_6: +; AVX1-NEXT: .LBB45_6: ; AVX1-NEXT: vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3] ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 ; AVX1-NEXT: vmovq %xmm0, %rax ; AVX1-NEXT: movl %eax, %ecx ; AVX1-NEXT: andl $1, %ecx ; AVX1-NEXT: testq %rax, %rax -; AVX1-NEXT: js .LBB29_7 +; AVX1-NEXT: js .LBB45_7 ; AVX1-NEXT: # BB#8: ; AVX1-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm2 -; AVX1-NEXT: jmp .LBB29_9 -; AVX1-NEXT: .LBB29_7: +; AVX1-NEXT: jmp .LBB45_9 +; AVX1-NEXT: .LBB45_7: ; AVX1-NEXT: shrq %rax ; AVX1-NEXT: orq %rax, %rcx ; AVX1-NEXT: vcvtsi2ssq %rcx, %xmm0, %xmm2 ; AVX1-NEXT: vaddss %xmm2, %xmm2, %xmm2 -; AVX1-NEXT: .LBB29_9: +; AVX1-NEXT: .LBB45_9: ; AVX1-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3] ; AVX1-NEXT: vpextrq $1, %xmm0, %rax ; AVX1-NEXT: movl %eax, %ecx ; AVX1-NEXT: andl $1, %ecx ; AVX1-NEXT: testq %rax, %rax -; AVX1-NEXT: js .LBB29_10 +; AVX1-NEXT: js .LBB45_10 ; AVX1-NEXT: # BB#11: ; AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0 ; AVX1-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm0 ; AVX1-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0] ; AVX1-NEXT: vzeroupper ; AVX1-NEXT: retq -; AVX1-NEXT: .LBB29_10: +; AVX1-NEXT: .LBB45_10: ; AVX1-NEXT: shrq %rax ; AVX1-NEXT: orq %rax, %rcx ; AVX1-NEXT: vcvtsi2ssq %rcx, %xmm0, %xmm0 @@ -1082,65 +1589,65 @@ define <4 x float> @uitofp_4vf32_4i64(<4 x i64> %a) { ; AVX1-NEXT: vzeroupper ; AVX1-NEXT: retq ; -; AVX2-LABEL: uitofp_4vf32_4i64: +; AVX2-LABEL: uitofp_4i64_to_4f32: ; AVX2: # BB#0: ; AVX2-NEXT: vpextrq $1, %xmm0, %rax ; AVX2-NEXT: movl %eax, %ecx ; AVX2-NEXT: andl $1, %ecx ; AVX2-NEXT: testq %rax, %rax -; AVX2-NEXT: js .LBB29_1 +; AVX2-NEXT: js .LBB45_1 ; AVX2-NEXT: # BB#2: ; AVX2-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm1 -; AVX2-NEXT: jmp .LBB29_3 -; AVX2-NEXT: .LBB29_1: +; AVX2-NEXT: jmp .LBB45_3 +; AVX2-NEXT: .LBB45_1: ; AVX2-NEXT: shrq %rax ; AVX2-NEXT: orq %rax, %rcx ; AVX2-NEXT: vcvtsi2ssq %rcx, %xmm0, %xmm1 ; AVX2-NEXT: vaddss %xmm1, %xmm1, %xmm1 -; AVX2-NEXT: .LBB29_3: +; AVX2-NEXT: .LBB45_3: ; AVX2-NEXT: vmovq %xmm0, %rax ; AVX2-NEXT: movl %eax, %ecx ; AVX2-NEXT: andl $1, %ecx ; AVX2-NEXT: testq %rax, %rax -; AVX2-NEXT: js .LBB29_4 +; AVX2-NEXT: js .LBB45_4 ; AVX2-NEXT: # BB#5: ; AVX2-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm2 -; AVX2-NEXT: jmp .LBB29_6 -; AVX2-NEXT: .LBB29_4: +; AVX2-NEXT: jmp .LBB45_6 +; AVX2-NEXT: .LBB45_4: ; AVX2-NEXT: shrq %rax ; AVX2-NEXT: orq %rax, %rcx ; AVX2-NEXT: vcvtsi2ssq %rcx, %xmm0, %xmm2 ; AVX2-NEXT: vaddss %xmm2, %xmm2, %xmm2 -; AVX2-NEXT: .LBB29_6: +; AVX2-NEXT: .LBB45_6: ; AVX2-NEXT: vinsertps {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[2,3] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 ; AVX2-NEXT: vmovq %xmm0, %rax ; AVX2-NEXT: movl %eax, %ecx ; AVX2-NEXT: andl $1, %ecx ; AVX2-NEXT: testq %rax, %rax -; AVX2-NEXT: js .LBB29_7 +; AVX2-NEXT: js .LBB45_7 ; AVX2-NEXT: # BB#8: ; AVX2-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm2 -; AVX2-NEXT: jmp .LBB29_9 -; AVX2-NEXT: .LBB29_7: +; AVX2-NEXT: jmp .LBB45_9 +; AVX2-NEXT: .LBB45_7: ; AVX2-NEXT: shrq %rax ; AVX2-NEXT: orq %rax, %rcx ; AVX2-NEXT: vcvtsi2ssq %rcx, %xmm0, %xmm2 ; AVX2-NEXT: vaddss %xmm2, %xmm2, %xmm2 -; AVX2-NEXT: .LBB29_9: +; AVX2-NEXT: .LBB45_9: ; AVX2-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3] ; AVX2-NEXT: vpextrq $1, %xmm0, %rax ; AVX2-NEXT: movl %eax, %ecx ; AVX2-NEXT: andl $1, %ecx ; AVX2-NEXT: testq %rax, %rax -; AVX2-NEXT: js .LBB29_10 +; AVX2-NEXT: js .LBB45_10 ; AVX2-NEXT: # BB#11: ; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0 ; AVX2-NEXT: vcvtsi2ssq %rax, %xmm0, %xmm0 ; AVX2-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0] ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq -; AVX2-NEXT: .LBB29_10: +; AVX2-NEXT: .LBB45_10: ; AVX2-NEXT: shrq %rax ; AVX2-NEXT: orq %rax, %rcx ; AVX2-NEXT: vcvtsi2ssq %rcx, %xmm0, %xmm0 @@ -1152,20 +1659,69 @@ define <4 x float> @uitofp_4vf32_4i64(<4 x i64> %a) { ret <4 x float> %cvt } -define <8 x float> @uitofp_8vf32_i16(<8 x i16> %a) { -; SSE2-LABEL: uitofp_8vf32_i16: -; SSE2: # BB#0: -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: movdqa %xmm0, %xmm2 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] -; SSE2-NEXT: cvtdq2ps %xmm2, %xmm2 -; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7] -; SSE2-NEXT: pand .LCPI30_0(%rip), %xmm0 -; SSE2-NEXT: cvtdq2ps %xmm0, %xmm1 -; SSE2-NEXT: movaps %xmm2, %xmm0 -; SSE2-NEXT: retq -; -; AVX1-LABEL: uitofp_8vf32_i16: +define <8 x float> @uitofp_8i32_to_8f32(<8 x i32> %a) { +; SSE-LABEL: uitofp_8i32_to_8f32: +; SSE: # BB#0: +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535] +; SSE-NEXT: movdqa %xmm0, %xmm3 +; SSE-NEXT: pand %xmm2, %xmm3 +; SSE-NEXT: movdqa {{.*#+}} xmm4 = [1258291200,1258291200,1258291200,1258291200] +; SSE-NEXT: por %xmm4, %xmm3 +; SSE-NEXT: psrld $16, %xmm0 +; SSE-NEXT: movdqa {{.*#+}} xmm5 = [1392508928,1392508928,1392508928,1392508928] +; SSE-NEXT: por %xmm5, %xmm0 +; SSE-NEXT: movaps {{.*#+}} xmm6 = [-5.497642e+11,-5.497642e+11,-5.497642e+11,-5.497642e+11] +; SSE-NEXT: addps %xmm6, %xmm0 +; SSE-NEXT: addps %xmm3, %xmm0 +; SSE-NEXT: pand %xmm1, %xmm2 +; SSE-NEXT: por %xmm4, %xmm2 +; SSE-NEXT: psrld $16, %xmm1 +; SSE-NEXT: por %xmm5, %xmm1 +; SSE-NEXT: addps %xmm6, %xmm1 +; SSE-NEXT: addps %xmm2, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: uitofp_8i32_to_8f32: +; AVX1: # BB#0: +; AVX1-NEXT: vandps {{.*}}(%rip), %ymm0, %ymm1 +; AVX1-NEXT: vcvtdq2ps %ymm1, %ymm1 +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0 +; AVX1-NEXT: vcvtdq2ps %ymm0, %ymm0 +; AVX1-NEXT: vmulps {{.*}}(%rip), %ymm0, %ymm0 +; AVX1-NEXT: vaddps %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: uitofp_8i32_to_8f32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %ymm1 +; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15] +; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0 +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7],ymm0[8],ymm2[9],ymm0[10],ymm2[11],ymm0[12],ymm2[13],ymm0[14],ymm2[15] +; AVX2-NEXT: vbroadcastss {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vaddps %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vaddps %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq + %cvt = uitofp <8 x i32> %a to <8 x float> + ret <8 x float> %cvt +} + +define <8 x float> @uitofp_8i16_to_8f32(<8 x i16> %a) { +; SSE-LABEL: uitofp_8i16_to_8f32: +; SSE: # BB#0: +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSE-NEXT: cvtdq2ps %xmm2, %xmm2 +; SSE-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: cvtdq2ps %xmm0, %xmm1 +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: uitofp_8i16_to_8f32: ; AVX1: # BB#0: ; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] @@ -1174,7 +1730,7 @@ define <8 x float> @uitofp_8vf32_i16(<8 x i16> %a) { ; AVX1-NEXT: vcvtdq2ps %ymm0, %ymm0 ; AVX1-NEXT: retq ; -; AVX2-LABEL: uitofp_8vf32_i16: +; AVX2-LABEL: uitofp_8i16_to_8f32: ; AVX2: # BB#0: ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero ; AVX2-NEXT: vcvtdq2ps %ymm0, %ymm0 @@ -1183,36 +1739,35 @@ define <8 x float> @uitofp_8vf32_i16(<8 x i16> %a) { ret <8 x float> %cvt } -define <8 x float> @uitofp_8vf32_i8(<16 x i8> %a) { -; SSE2-LABEL: uitofp_8vf32_i8: -; SSE2: # BB#0: -; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: movdqa %xmm0, %xmm2 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] -; SSE2-NEXT: cvtdq2ps %xmm2, %xmm2 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7] -; SSE2-NEXT: pand .LCPI31_0(%rip), %xmm0 -; SSE2-NEXT: cvtdq2ps %xmm0, %xmm1 -; SSE2-NEXT: movaps %xmm2, %xmm0 -; SSE2-NEXT: retq -; -; AVX1-LABEL: uitofp_8vf32_i8: +define <8 x float> @uitofp_8i8_to_8f32(<16 x i8> %a) { +; SSE-LABEL: uitofp_8i8_to_8f32: +; SSE: # BB#0: +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSE-NEXT: cvtdq2ps %xmm2, %xmm2 +; SSE-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: cvtdq2ps %xmm0, %xmm1 +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: uitofp_8i8_to_8f32: ; AVX1: # BB#0: ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0] +; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1 ; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; AVX1-NEXT: vandps .LCPI31_0(%rip), %ymm0, %ymm0 ; AVX1-NEXT: vcvtdq2ps %ymm0, %ymm0 ; AVX1-NEXT: retq ; -; AVX2-LABEL: uitofp_8vf32_i8: +; AVX2-LABEL: uitofp_8i8_to_8f32: ; AVX2: # BB#0: ; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero -; AVX2-NEXT: vpbroadcastd .LCPI31_0(%rip), %ymm1 -; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 ; AVX2-NEXT: vcvtdq2ps %ymm0, %ymm0 ; AVX2-NEXT: retq %shuf = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> @@ -1220,28 +1775,61 @@ define <8 x float> @uitofp_8vf32_i8(<16 x i8> %a) { ret <8 x float> %cvt } +define <8 x float> @uitofp_16i8_to_8f32(<16 x i8> %a) { +; SSE-LABEL: uitofp_16i8_to_8f32: +; SSE: # BB#0: +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSE-NEXT: cvtdq2ps %xmm2, %xmm2 +; SSE-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: cvtdq2ps %xmm0, %xmm1 +; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: uitofp_16i8_to_8f32: +; AVX1: # BB#0: +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vcvtdq2ps %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: uitofp_16i8_to_8f32: +; AVX2: # BB#0: +; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX2-NEXT: vcvtdq2ps %ymm0, %ymm0 +; AVX2-NEXT: retq + %cvt = uitofp <16 x i8> %a to <16 x float> + %shuf = shufflevector <16 x float> %cvt, <16 x float> undef, <8 x i32> + ret <8 x float> %shuf +} + ; ; Aggregates ; %Arguments = type <{ <8 x i8>, <8 x i16>, <8 x float>* }> -define void @aggregate_sitofp_8f32_i16(%Arguments* nocapture readonly %a0) { -; SSE2-LABEL: aggregate_sitofp_8f32_i16: -; SSE2: # BB#0: -; SSE2-NEXT: movq 24(%rdi), %rax -; SSE2-NEXT: movdqu 8(%rdi), %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] -; SSE2-NEXT: psrad $16, %xmm1 -; SSE2-NEXT: cvtdq2ps %xmm1, %xmm1 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] -; SSE2-NEXT: psrad $16, %xmm0 -; SSE2-NEXT: cvtdq2ps %xmm0, %xmm0 -; SSE2-NEXT: movaps %xmm0, (%rax) -; SSE2-NEXT: movaps %xmm1, 16(%rax) -; SSE2-NEXT: retq -; -; AVX1-LABEL: aggregate_sitofp_8f32_i16: +define void @aggregate_sitofp_8i16_to_8f32(%Arguments* nocapture readonly %a0) { +; SSE-LABEL: aggregate_sitofp_8i16_to_8f32: +; SSE: # BB#0: +; SSE-NEXT: movq 24(%rdi), %rax +; SSE-NEXT: movdqu 8(%rdi), %xmm0 +; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSE-NEXT: psrad $16, %xmm1 +; SSE-NEXT: cvtdq2ps %xmm1, %xmm1 +; SSE-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7] +; SSE-NEXT: psrad $16, %xmm0 +; SSE-NEXT: cvtdq2ps %xmm0, %xmm0 +; SSE-NEXT: movaps %xmm0, 16(%rax) +; SSE-NEXT: movaps %xmm1, (%rax) +; SSE-NEXT: retq +; +; AVX1-LABEL: aggregate_sitofp_8i16_to_8f32: ; AVX1: # BB#0: ; AVX1-NEXT: movq 24(%rdi), %rax ; AVX1-NEXT: vmovdqu 8(%rdi), %xmm0 @@ -1254,7 +1842,7 @@ define void @aggregate_sitofp_8f32_i16(%Arguments* nocapture readonly %a0) { ; AVX1-NEXT: vzeroupper ; AVX1-NEXT: retq ; -; AVX2-LABEL: aggregate_sitofp_8f32_i16: +; AVX2-LABEL: aggregate_sitofp_8i16_to_8f32: ; AVX2: # BB#0: ; AVX2-NEXT: movq 24(%rdi), %rax ; AVX2-NEXT: vpmovsxwd 8(%rdi), %ymm0 diff --git a/test/CodeGen/X86/vec_minmax_sint.ll b/test/CodeGen/X86/vec_minmax_sint.ll new file mode 100644 index 000000000000..419eb2bed743 --- /dev/null +++ b/test/CodeGen/X86/vec_minmax_sint.ll @@ -0,0 +1,2090 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE42 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW + +; +; Signed Maximum (GT) +; + +define <2 x i64> @max_gt_v2i64(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: max_gt_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_gt_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,0,2147483648,0] +; SSE41-NEXT: movdqa %xmm1, %xmm3 +; SSE41-NEXT: pxor %xmm0, %xmm3 +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm5, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: por %xmm3, %xmm0 +; SSE41-NEXT: blendvpd %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_gt_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa %xmm0, %xmm2 +; SSE42-NEXT: pcmpgtq %xmm1, %xmm0 +; SSE42-NEXT: blendvpd %xmm2, %xmm1 +; SSE42-NEXT: movapd %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: max_gt_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm2 +; AVX-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq + %1 = icmp sgt <2 x i64> %a, %b + %2 = select <2 x i1> %1, <2 x i64> %a, <2 x i64> %b + ret <2 x i64> %2 +} + +define <4 x i64> @max_gt_v4i64(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: max_gt_v4i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm3, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm1, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: movdqa %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm5, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm2, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: pxor %xmm0, %xmm4 +; SSE2-NEXT: movdqa %xmm4, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm5, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: pand %xmm6, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm6 +; SSE2-NEXT: por %xmm6, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_gt_v4i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm8 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,0,2147483648,0] +; SSE41-NEXT: movdqa %xmm3, %xmm5 +; SSE41-NEXT: pxor %xmm0, %xmm5 +; SSE41-NEXT: movdqa %xmm1, %xmm6 +; SSE41-NEXT: pxor %xmm0, %xmm6 +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] +; SSE41-NEXT: pand %xmm4, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE41-NEXT: por %xmm6, %xmm5 +; SSE41-NEXT: movdqa %xmm2, %xmm4 +; SSE41-NEXT: pxor %xmm0, %xmm4 +; SSE41-NEXT: pxor %xmm8, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm7, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] +; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: blendvpd %xmm8, %xmm2 +; SSE41-NEXT: movdqa %xmm5, %xmm0 +; SSE41-NEXT: blendvpd %xmm1, %xmm3 +; SSE41-NEXT: movapd %xmm2, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_gt_v4i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa %xmm0, %xmm4 +; SSE42-NEXT: movdqa %xmm1, %xmm5 +; SSE42-NEXT: pcmpgtq %xmm3, %xmm5 +; SSE42-NEXT: pcmpgtq %xmm2, %xmm0 +; SSE42-NEXT: blendvpd %xmm4, %xmm2 +; SSE42-NEXT: movdqa %xmm5, %xmm0 +; SSE42-NEXT: blendvpd %xmm1, %xmm3 +; SSE42-NEXT: movapd %xmm2, %xmm0 +; SSE42-NEXT: movapd %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: max_gt_v4i64: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: max_gt_v4i64: +; AVX2: # BB#0: +; AVX2-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: max_gt_v4i64: +; AVX512: # BB#0: +; AVX512-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm2 +; AVX512-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX512-NEXT: retq + %1 = icmp sgt <4 x i64> %a, %b + %2 = select <4 x i1> %1, <4 x i64> %a, <4 x i64> %b + ret <4 x i64> %2 +} + +define <4 x i32> @max_gt_v4i32(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: max_gt_v4i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_gt_v4i32: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxsd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_gt_v4i32: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxsd %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: max_gt_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp sgt <4 x i32> %a, %b + %2 = select <4 x i1> %1, <4 x i32> %a, <4 x i32> %b + ret <4 x i32> %2 +} + +define <8 x i32> @max_gt_v8i32(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: max_gt_v8i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm2, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm0, %xmm5 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm1, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_gt_v8i32: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxsd %xmm2, %xmm0 +; SSE41-NEXT: pmaxsd %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_gt_v8i32: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxsd %xmm2, %xmm0 +; SSE42-NEXT: pmaxsd %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: max_gt_v8i32: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsd %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: max_gt_v8i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: max_gt_v8i32: +; AVX512: # BB#0: +; AVX512-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp sgt <8 x i32> %a, %b + %2 = select <8 x i1> %1, <8 x i32> %a, <8 x i32> %b + ret <8 x i32> %2 +} + +define <8 x i16> @max_gt_v8i16(<8 x i16> %a, <8 x i16> %b) { +; SSE-LABEL: max_gt_v8i16: +; SSE: # BB#0: +; SSE-NEXT: pmaxsw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: max_gt_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp sgt <8 x i16> %a, %b + %2 = select <8 x i1> %1, <8 x i16> %a, <8 x i16> %b + ret <8 x i16> %2 +} + +define <16 x i16> @max_gt_v16i16(<16 x i16> %a, <16 x i16> %b) { +; SSE-LABEL: max_gt_v16i16: +; SSE: # BB#0: +; SSE-NEXT: pmaxsw %xmm2, %xmm0 +; SSE-NEXT: pmaxsw %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: max_gt_v16i16: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: max_gt_v16i16: +; AVX2: # BB#0: +; AVX2-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: max_gt_v16i16: +; AVX512: # BB#0: +; AVX512-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp sgt <16 x i16> %a, %b + %2 = select <16 x i1> %1, <16 x i16> %a, <16 x i16> %b + ret <16 x i16> %2 +} + +define <16 x i8> @max_gt_v16i8(<16 x i8> %a, <16 x i8> %b) { +; SSE2-LABEL: max_gt_v16i8: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_gt_v16i8: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxsb %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_gt_v16i8: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxsb %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: max_gt_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp sgt <16 x i8> %a, %b + %2 = select <16 x i1> %1, <16 x i8> %a, <16 x i8> %b + ret <16 x i8> %2 +} + +define <32 x i8> @max_gt_v32i8(<32 x i8> %a, <32 x i8> %b) { +; SSE2-LABEL: max_gt_v32i8: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pcmpgtb %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pcmpgtb %xmm2, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm0, %xmm5 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm1, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_gt_v32i8: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxsb %xmm2, %xmm0 +; SSE41-NEXT: pmaxsb %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_gt_v32i8: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxsb %xmm2, %xmm0 +; SSE42-NEXT: pmaxsb %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: max_gt_v32i8: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsb %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: max_gt_v32i8: +; AVX2: # BB#0: +; AVX2-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: max_gt_v32i8: +; AVX512: # BB#0: +; AVX512-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp sgt <32 x i8> %a, %b + %2 = select <32 x i1> %1, <32 x i8> %a, <32 x i8> %b + ret <32 x i8> %2 +} + +; +; Signed Maximum (GE) +; + +define <2 x i64> @max_ge_v2i64(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: max_ge_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_ge_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,0,2147483648,0] +; SSE41-NEXT: movdqa %xmm2, %xmm3 +; SSE41-NEXT: pxor %xmm0, %xmm3 +; SSE41-NEXT: pxor %xmm1, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE41-NEXT: por %xmm0, %xmm3 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: blendvpd %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_ge_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa %xmm0, %xmm2 +; SSE42-NEXT: movdqa %xmm1, %xmm3 +; SSE42-NEXT: pcmpgtq %xmm2, %xmm3 +; SSE42-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE42-NEXT: pxor %xmm3, %xmm0 +; SSE42-NEXT: blendvpd %xmm2, %xmm1 +; SSE42-NEXT: movapd %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: max_ge_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm2 +; AVX-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq + %1 = icmp sge <2 x i64> %a, %b + %2 = select <2 x i1> %1, <2 x i64> %a, <2 x i64> %b + ret <2 x i64> %2 +} + +define <4 x i64> @max_ge_v4i64(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: max_ge_v4i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm3, %xmm5 +; SSE2-NEXT: pxor %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm8 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm8, %xmm9 +; SSE2-NEXT: pxor %xmm4, %xmm9 +; SSE2-NEXT: movdqa %xmm0, %xmm6 +; SSE2-NEXT: pxor %xmm7, %xmm6 +; SSE2-NEXT: pxor %xmm2, %xmm7 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm5[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: pand %xmm10, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm1, %xmm8 +; SSE2-NEXT: pandn %xmm3, %xmm9 +; SSE2-NEXT: por %xmm8, %xmm9 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm9, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_ge_v4i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm8 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,0,2147483648,0] +; SSE41-NEXT: movdqa %xmm1, %xmm5 +; SSE41-NEXT: pxor %xmm0, %xmm5 +; SSE41-NEXT: movdqa %xmm3, %xmm6 +; SSE41-NEXT: pxor %xmm0, %xmm6 +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] +; SSE41-NEXT: pand %xmm4, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE41-NEXT: por %xmm6, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm9 +; SSE41-NEXT: pxor %xmm9, %xmm5 +; SSE41-NEXT: movdqa %xmm8, %xmm6 +; SSE41-NEXT: pxor %xmm0, %xmm6 +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm4, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: pxor %xmm9, %xmm0 +; SSE41-NEXT: blendvpd %xmm8, %xmm2 +; SSE41-NEXT: movdqa %xmm5, %xmm0 +; SSE41-NEXT: blendvpd %xmm1, %xmm3 +; SSE41-NEXT: movapd %xmm2, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_ge_v4i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa %xmm0, %xmm4 +; SSE42-NEXT: movdqa %xmm3, %xmm5 +; SSE42-NEXT: pcmpgtq %xmm1, %xmm5 +; SSE42-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE42-NEXT: pxor %xmm0, %xmm5 +; SSE42-NEXT: movdqa %xmm2, %xmm6 +; SSE42-NEXT: pcmpgtq %xmm4, %xmm6 +; SSE42-NEXT: pxor %xmm6, %xmm0 +; SSE42-NEXT: blendvpd %xmm4, %xmm2 +; SSE42-NEXT: movdqa %xmm5, %xmm0 +; SSE42-NEXT: blendvpd %xmm1, %xmm3 +; SSE42-NEXT: movapd %xmm2, %xmm0 +; SSE42-NEXT: movapd %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: max_ge_v4i64: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm4 +; AVX1-NEXT: vpxor %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: max_ge_v4i64: +; AVX2: # BB#0: +; AVX2-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm2 +; AVX2-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3 +; AVX2-NEXT: vpxor %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: max_ge_v4i64: +; AVX512: # BB#0: +; AVX512-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm2 +; AVX512-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3 +; AVX512-NEXT: vpxor %ymm3, %ymm2, %ymm2 +; AVX512-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX512-NEXT: retq + %1 = icmp sge <4 x i64> %a, %b + %2 = select <4 x i1> %1, <4 x i64> %a, <4 x i64> %b + ret <4 x i64> %2 +} + +define <4 x i32> @max_ge_v4i32(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: max_ge_v4i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_ge_v4i32: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxsd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_ge_v4i32: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxsd %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: max_ge_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp sge <4 x i32> %a, %b + %2 = select <4 x i1> %1, <4 x i32> %a, <4 x i32> %b + ret <4 x i32> %2 +} + +define <8 x i32> @max_ge_v8i32(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: max_ge_v8i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm6 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm2, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm7 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm7 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm7, %xmm4 +; SSE2-NEXT: pandn %xmm1, %xmm6 +; SSE2-NEXT: pandn %xmm3, %xmm5 +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_ge_v8i32: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxsd %xmm2, %xmm0 +; SSE41-NEXT: pmaxsd %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_ge_v8i32: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxsd %xmm2, %xmm0 +; SSE42-NEXT: pmaxsd %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: max_ge_v8i32: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsd %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: max_ge_v8i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: max_ge_v8i32: +; AVX512: # BB#0: +; AVX512-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp sge <8 x i32> %a, %b + %2 = select <8 x i1> %1, <8 x i32> %a, <8 x i32> %b + ret <8 x i32> %2 +} + +define <8 x i16> @max_ge_v8i16(<8 x i16> %a, <8 x i16> %b) { +; SSE-LABEL: max_ge_v8i16: +; SSE: # BB#0: +; SSE-NEXT: pmaxsw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: max_ge_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp sge <8 x i16> %a, %b + %2 = select <8 x i1> %1, <8 x i16> %a, <8 x i16> %b + ret <8 x i16> %2 +} + +define <16 x i16> @max_ge_v16i16(<16 x i16> %a, <16 x i16> %b) { +; SSE-LABEL: max_ge_v16i16: +; SSE: # BB#0: +; SSE-NEXT: pmaxsw %xmm2, %xmm0 +; SSE-NEXT: pmaxsw %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: max_ge_v16i16: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: max_ge_v16i16: +; AVX2: # BB#0: +; AVX2-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: max_ge_v16i16: +; AVX512: # BB#0: +; AVX512-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp sge <16 x i16> %a, %b + %2 = select <16 x i1> %1, <16 x i16> %a, <16 x i16> %b + ret <16 x i16> %2 +} + +define <16 x i8> @max_ge_v16i8(<16 x i8> %a, <16 x i8> %b) { +; SSE2-LABEL: max_ge_v16i8: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pcmpgtb %xmm0, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_ge_v16i8: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxsb %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_ge_v16i8: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxsb %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: max_ge_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp sge <16 x i8> %a, %b + %2 = select <16 x i1> %1, <16 x i8> %a, <16 x i8> %b + ret <16 x i8> %2 +} + +define <32 x i8> @max_ge_v32i8(<32 x i8> %a, <32 x i8> %b) { +; SSE2-LABEL: max_ge_v32i8: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm6 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm2, %xmm7 +; SSE2-NEXT: pcmpgtb %xmm0, %xmm7 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm7 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm7, %xmm4 +; SSE2-NEXT: pandn %xmm1, %xmm6 +; SSE2-NEXT: pandn %xmm3, %xmm5 +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_ge_v32i8: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxsb %xmm2, %xmm0 +; SSE41-NEXT: pmaxsb %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_ge_v32i8: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxsb %xmm2, %xmm0 +; SSE42-NEXT: pmaxsb %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: max_ge_v32i8: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsb %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: max_ge_v32i8: +; AVX2: # BB#0: +; AVX2-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: max_ge_v32i8: +; AVX512: # BB#0: +; AVX512-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp sge <32 x i8> %a, %b + %2 = select <32 x i1> %1, <32 x i8> %a, <32 x i8> %b + ret <32 x i8> %2 +} + +; +; Signed Minimum (LT) +; + +define <2 x i64> @min_lt_v2i64(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: min_lt_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_lt_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,0,2147483648,0] +; SSE41-NEXT: movdqa %xmm2, %xmm3 +; SSE41-NEXT: pxor %xmm0, %xmm3 +; SSE41-NEXT: pxor %xmm1, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm5, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: por %xmm3, %xmm0 +; SSE41-NEXT: blendvpd %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_lt_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa %xmm0, %xmm2 +; SSE42-NEXT: movdqa %xmm1, %xmm0 +; SSE42-NEXT: pcmpgtq %xmm2, %xmm0 +; SSE42-NEXT: blendvpd %xmm2, %xmm1 +; SSE42-NEXT: movapd %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: min_lt_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm2 +; AVX-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq + %1 = icmp slt <2 x i64> %a, %b + %2 = select <2 x i1> %1, <2 x i64> %a, <2 x i64> %b + ret <2 x i64> %2 +} + +define <4 x i64> @min_lt_v4i64(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: min_lt_v4i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: movdqa %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm5, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: pxor %xmm2, %xmm4 +; SSE2-NEXT: movdqa %xmm4, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm5, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: pand %xmm6, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm6 +; SSE2-NEXT: por %xmm6, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_lt_v4i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm8 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,0,2147483648,0] +; SSE41-NEXT: movdqa %xmm1, %xmm5 +; SSE41-NEXT: pxor %xmm0, %xmm5 +; SSE41-NEXT: movdqa %xmm3, %xmm6 +; SSE41-NEXT: pxor %xmm0, %xmm6 +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] +; SSE41-NEXT: pand %xmm4, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE41-NEXT: por %xmm6, %xmm5 +; SSE41-NEXT: movdqa %xmm8, %xmm4 +; SSE41-NEXT: pxor %xmm0, %xmm4 +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm7, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] +; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: blendvpd %xmm8, %xmm2 +; SSE41-NEXT: movdqa %xmm5, %xmm0 +; SSE41-NEXT: blendvpd %xmm1, %xmm3 +; SSE41-NEXT: movapd %xmm2, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_lt_v4i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa %xmm0, %xmm4 +; SSE42-NEXT: movdqa %xmm3, %xmm5 +; SSE42-NEXT: pcmpgtq %xmm1, %xmm5 +; SSE42-NEXT: movdqa %xmm2, %xmm0 +; SSE42-NEXT: pcmpgtq %xmm4, %xmm0 +; SSE42-NEXT: blendvpd %xmm4, %xmm2 +; SSE42-NEXT: movdqa %xmm5, %xmm0 +; SSE42-NEXT: blendvpd %xmm1, %xmm3 +; SSE42-NEXT: movapd %xmm2, %xmm0 +; SSE42-NEXT: movapd %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: min_lt_v4i64: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: min_lt_v4i64: +; AVX2: # BB#0: +; AVX2-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: min_lt_v4i64: +; AVX512: # BB#0: +; AVX512-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm2 +; AVX512-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX512-NEXT: retq + %1 = icmp slt <4 x i64> %a, %b + %2 = select <4 x i1> %1, <4 x i64> %a, <4 x i64> %b + ret <4 x i64> %2 +} + +define <4 x i32> @min_lt_v4i32(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: min_lt_v4i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_lt_v4i32: +; SSE41: # BB#0: +; SSE41-NEXT: pminsd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_lt_v4i32: +; SSE42: # BB#0: +; SSE42-NEXT: pminsd %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: min_lt_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpminsd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp slt <4 x i32> %a, %b + %2 = select <4 x i1> %1, <4 x i32> %a, <4 x i32> %b + ret <4 x i32> %2 +} + +define <8 x i32> @min_lt_v8i32(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: min_lt_v8i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm4 +; SSE2-NEXT: movdqa %xmm2, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_lt_v8i32: +; SSE41: # BB#0: +; SSE41-NEXT: pminsd %xmm2, %xmm0 +; SSE41-NEXT: pminsd %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_lt_v8i32: +; SSE42: # BB#0: +; SSE42-NEXT: pminsd %xmm2, %xmm0 +; SSE42-NEXT: pminsd %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: min_lt_v8i32: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsd %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: min_lt_v8i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpminsd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: min_lt_v8i32: +; AVX512: # BB#0: +; AVX512-NEXT: vpminsd %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp slt <8 x i32> %a, %b + %2 = select <8 x i1> %1, <8 x i32> %a, <8 x i32> %b + ret <8 x i32> %2 +} + +define <8 x i16> @min_lt_v8i16(<8 x i16> %a, <8 x i16> %b) { +; SSE-LABEL: min_lt_v8i16: +; SSE: # BB#0: +; SSE-NEXT: pminsw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: min_lt_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpminsw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp slt <8 x i16> %a, %b + %2 = select <8 x i1> %1, <8 x i16> %a, <8 x i16> %b + ret <8 x i16> %2 +} + +define <16 x i16> @min_lt_v16i16(<16 x i16> %a, <16 x i16> %b) { +; SSE-LABEL: min_lt_v16i16: +; SSE: # BB#0: +; SSE-NEXT: pminsw %xmm2, %xmm0 +; SSE-NEXT: pminsw %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: min_lt_v16i16: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: min_lt_v16i16: +; AVX2: # BB#0: +; AVX2-NEXT: vpminsw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: min_lt_v16i16: +; AVX512: # BB#0: +; AVX512-NEXT: vpminsw %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp slt <16 x i16> %a, %b + %2 = select <16 x i1> %1, <16 x i16> %a, <16 x i16> %b + ret <16 x i16> %2 +} + +define <16 x i8> @min_lt_v16i8(<16 x i8> %a, <16 x i8> %b) { +; SSE2-LABEL: min_lt_v16i8: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtb %xmm0, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_lt_v16i8: +; SSE41: # BB#0: +; SSE41-NEXT: pminsb %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_lt_v16i8: +; SSE42: # BB#0: +; SSE42-NEXT: pminsb %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: min_lt_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpminsb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp slt <16 x i8> %a, %b + %2 = select <16 x i1> %1, <16 x i8> %a, <16 x i8> %b + ret <16 x i8> %2 +} + +define <32 x i8> @min_lt_v32i8(<32 x i8> %a, <32 x i8> %b) { +; SSE2-LABEL: min_lt_v32i8: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm4 +; SSE2-NEXT: movdqa %xmm2, %xmm5 +; SSE2-NEXT: pcmpgtb %xmm0, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_lt_v32i8: +; SSE41: # BB#0: +; SSE41-NEXT: pminsb %xmm2, %xmm0 +; SSE41-NEXT: pminsb %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_lt_v32i8: +; SSE42: # BB#0: +; SSE42-NEXT: pminsb %xmm2, %xmm0 +; SSE42-NEXT: pminsb %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: min_lt_v32i8: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsb %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: min_lt_v32i8: +; AVX2: # BB#0: +; AVX2-NEXT: vpminsb %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: min_lt_v32i8: +; AVX512: # BB#0: +; AVX512-NEXT: vpminsb %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp slt <32 x i8> %a, %b + %2 = select <32 x i1> %1, <32 x i8> %a, <32 x i8> %b + ret <32 x i8> %2 +} + +; +; Signed Minimum (LE) +; + +define <2 x i64> @min_le_v2i64(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: min_le_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_le_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,0,2147483648,0] +; SSE41-NEXT: movdqa %xmm1, %xmm3 +; SSE41-NEXT: pxor %xmm0, %xmm3 +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE41-NEXT: por %xmm0, %xmm3 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: blendvpd %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_le_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa %xmm0, %xmm2 +; SSE42-NEXT: pcmpgtq %xmm1, %xmm0 +; SSE42-NEXT: pcmpeqd %xmm3, %xmm3 +; SSE42-NEXT: pxor %xmm3, %xmm0 +; SSE42-NEXT: blendvpd %xmm2, %xmm1 +; SSE42-NEXT: movapd %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: min_le_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm2 +; AVX-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq + %1 = icmp sle <2 x i64> %a, %b + %2 = select <2 x i1> %1, <2 x i64> %a, <2 x i64> %b + ret <2 x i64> %2 +} + +define <4 x i64> @min_le_v4i64(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: min_le_v4i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pxor %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm8 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm8, %xmm9 +; SSE2-NEXT: pxor %xmm4, %xmm9 +; SSE2-NEXT: movdqa %xmm2, %xmm6 +; SSE2-NEXT: pxor %xmm7, %xmm6 +; SSE2-NEXT: pxor %xmm0, %xmm7 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm5[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: pand %xmm10, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm1, %xmm8 +; SSE2-NEXT: pandn %xmm3, %xmm9 +; SSE2-NEXT: por %xmm8, %xmm9 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm9, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_le_v4i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm8 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,0,2147483648,0] +; SSE41-NEXT: movdqa %xmm3, %xmm5 +; SSE41-NEXT: pxor %xmm0, %xmm5 +; SSE41-NEXT: movdqa %xmm1, %xmm6 +; SSE41-NEXT: pxor %xmm0, %xmm6 +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] +; SSE41-NEXT: pand %xmm4, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE41-NEXT: por %xmm6, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm9 +; SSE41-NEXT: pxor %xmm9, %xmm5 +; SSE41-NEXT: movdqa %xmm2, %xmm6 +; SSE41-NEXT: pxor %xmm0, %xmm6 +; SSE41-NEXT: pxor %xmm8, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm4, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: pxor %xmm9, %xmm0 +; SSE41-NEXT: blendvpd %xmm8, %xmm2 +; SSE41-NEXT: movdqa %xmm5, %xmm0 +; SSE41-NEXT: blendvpd %xmm1, %xmm3 +; SSE41-NEXT: movapd %xmm2, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_le_v4i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa %xmm0, %xmm4 +; SSE42-NEXT: movdqa %xmm1, %xmm5 +; SSE42-NEXT: pcmpgtq %xmm3, %xmm5 +; SSE42-NEXT: pcmpeqd %xmm6, %xmm6 +; SSE42-NEXT: pxor %xmm6, %xmm5 +; SSE42-NEXT: pcmpgtq %xmm2, %xmm0 +; SSE42-NEXT: pxor %xmm6, %xmm0 +; SSE42-NEXT: blendvpd %xmm4, %xmm2 +; SSE42-NEXT: movdqa %xmm5, %xmm0 +; SSE42-NEXT: blendvpd %xmm1, %xmm3 +; SSE42-NEXT: movapd %xmm2, %xmm0 +; SSE42-NEXT: movapd %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: min_le_v4i64: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm4 +; AVX1-NEXT: vpxor %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: min_le_v4i64: +; AVX2: # BB#0: +; AVX2-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm2 +; AVX2-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3 +; AVX2-NEXT: vpxor %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: min_le_v4i64: +; AVX512: # BB#0: +; AVX512-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm2 +; AVX512-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3 +; AVX512-NEXT: vpxor %ymm3, %ymm2, %ymm2 +; AVX512-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX512-NEXT: retq + %1 = icmp sle <4 x i64> %a, %b + %2 = select <4 x i1> %1, <4 x i64> %a, <4 x i64> %b + ret <4 x i64> %2 +} + +define <4 x i32> @min_le_v4i32(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: min_le_v4i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE2-NEXT: pcmpeqd %xmm3, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_le_v4i32: +; SSE41: # BB#0: +; SSE41-NEXT: pminsd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_le_v4i32: +; SSE42: # BB#0: +; SSE42-NEXT: pminsd %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: min_le_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpminsd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp sle <4 x i32> %a, %b + %2 = select <4 x i1> %1, <4 x i32> %a, <4 x i32> %b + ret <4 x i32> %2 +} + +define <8 x i32> @min_le_v8i32(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: min_le_v8i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm1, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm6 +; SSE2-NEXT: pcmpeqd %xmm7, %xmm7 +; SSE2-NEXT: movdqa %xmm6, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm2, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm7 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: pandn %xmm2, %xmm7 +; SSE2-NEXT: por %xmm7, %xmm5 +; SSE2-NEXT: pandn %xmm1, %xmm6 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_le_v8i32: +; SSE41: # BB#0: +; SSE41-NEXT: pminsd %xmm2, %xmm0 +; SSE41-NEXT: pminsd %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_le_v8i32: +; SSE42: # BB#0: +; SSE42-NEXT: pminsd %xmm2, %xmm0 +; SSE42-NEXT: pminsd %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: min_le_v8i32: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsd %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: min_le_v8i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpminsd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: min_le_v8i32: +; AVX512: # BB#0: +; AVX512-NEXT: vpminsd %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp sle <8 x i32> %a, %b + %2 = select <8 x i1> %1, <8 x i32> %a, <8 x i32> %b + ret <8 x i32> %2 +} + +define <8 x i16> @min_le_v8i16(<8 x i16> %a, <8 x i16> %b) { +; SSE-LABEL: min_le_v8i16: +; SSE: # BB#0: +; SSE-NEXT: pminsw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: min_le_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpminsw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp sle <8 x i16> %a, %b + %2 = select <8 x i1> %1, <8 x i16> %a, <8 x i16> %b + ret <8 x i16> %2 +} + +define <16 x i16> @min_le_v16i16(<16 x i16> %a, <16 x i16> %b) { +; SSE-LABEL: min_le_v16i16: +; SSE: # BB#0: +; SSE-NEXT: pminsw %xmm2, %xmm0 +; SSE-NEXT: pminsw %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: min_le_v16i16: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: min_le_v16i16: +; AVX2: # BB#0: +; AVX2-NEXT: vpminsw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: min_le_v16i16: +; AVX512: # BB#0: +; AVX512-NEXT: vpminsw %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp sle <16 x i16> %a, %b + %2 = select <16 x i1> %1, <16 x i16> %a, <16 x i16> %b + ret <16 x i16> %2 +} + +define <16 x i8> @min_le_v16i8(<16 x i8> %a, <16 x i8> %b) { +; SSE2-LABEL: min_le_v16i8: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm2 +; SSE2-NEXT: pcmpeqd %xmm3, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_le_v16i8: +; SSE41: # BB#0: +; SSE41-NEXT: pminsb %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_le_v16i8: +; SSE42: # BB#0: +; SSE42-NEXT: pminsb %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: min_le_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpminsb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp sle <16 x i8> %a, %b + %2 = select <16 x i1> %1, <16 x i8> %a, <16 x i8> %b + ret <16 x i8> %2 +} + +define <32 x i8> @min_le_v32i8(<32 x i8> %a, <32 x i8> %b) { +; SSE2-LABEL: min_le_v32i8: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm1, %xmm6 +; SSE2-NEXT: pcmpgtb %xmm3, %xmm6 +; SSE2-NEXT: pcmpeqd %xmm7, %xmm7 +; SSE2-NEXT: movdqa %xmm6, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pcmpgtb %xmm2, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm7 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: pandn %xmm2, %xmm7 +; SSE2-NEXT: por %xmm7, %xmm5 +; SSE2-NEXT: pandn %xmm1, %xmm6 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_le_v32i8: +; SSE41: # BB#0: +; SSE41-NEXT: pminsb %xmm2, %xmm0 +; SSE41-NEXT: pminsb %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_le_v32i8: +; SSE42: # BB#0: +; SSE42-NEXT: pminsb %xmm2, %xmm0 +; SSE42-NEXT: pminsb %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: min_le_v32i8: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsb %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: min_le_v32i8: +; AVX2: # BB#0: +; AVX2-NEXT: vpminsb %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: min_le_v32i8: +; AVX512: # BB#0: +; AVX512-NEXT: vpminsb %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp sle <32 x i8> %a, %b + %2 = select <32 x i1> %1, <32 x i8> %a, <32 x i8> %b + ret <32 x i8> %2 +} + +; +; Constant Folding +; + +define <2 x i64> @max_gt_v2i64c() { +; SSE-LABEL: max_gt_v2i64c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [18446744073709551615,7] +; SSE-NEXT: retq +; +; AVX-LABEL: max_gt_v2i64c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [18446744073709551615,7] +; AVX-NEXT: retq + %1 = insertelement <2 x i64> , i64 -7, i32 0 + %2 = insertelement <2 x i64> , i64 -1, i32 0 + %3 = icmp sgt <2 x i64> %1, %2 + %4 = select <2 x i1> %3, <2 x i64> %1, <2 x i64> %2 + ret <2 x i64> %4 +} + +define <4 x i64> @max_gt_v4i64c() { +; SSE-LABEL: max_gt_v4i64c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm1 = [7,7] +; SSE-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: max_gt_v4i64c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [18446744073709551615,18446744073709551615,7,7] +; AVX-NEXT: retq + %1 = insertelement <4 x i64> , i64 -7, i32 0 + %2 = insertelement <4 x i64> , i64 -1, i32 0 + %3 = icmp sgt <4 x i64> %1, %2 + %4 = select <4 x i1> %3, <4 x i64> %1, <4 x i64> %2 + ret <4 x i64> %4 +} + +define <4 x i32> @max_gt_v4i32c() { +; SSE-LABEL: max_gt_v4i32c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [4294967295,4294967295,7,7] +; SSE-NEXT: retq +; +; AVX-LABEL: max_gt_v4i32c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [4294967295,4294967295,7,7] +; AVX-NEXT: retq + %1 = insertelement <4 x i32> , i32 -7, i32 0 + %2 = insertelement <4 x i32> , i32 -1, i32 0 + %3 = icmp sgt <4 x i32> %1, %2 + %4 = select <4 x i1> %3, <4 x i32> %1, <4 x i32> %2 + ret <4 x i32> %4 +} + +define <8 x i32> @max_gt_v8i32c() { +; SSE-LABEL: max_gt_v8i32c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [4294967295,4294967293,4294967293,4294967295] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [7,5,5,7] +; SSE-NEXT: retq +; +; AVX-LABEL: max_gt_v8i32c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [4294967295,4294967293,4294967293,4294967295,7,5,5,7] +; AVX-NEXT: retq + %1 = insertelement <8 x i32> , i32 -7, i32 0 + %2 = insertelement <8 x i32> , i32 -1, i32 0 + %3 = icmp sgt <8 x i32> %1, %2 + %4 = select <8 x i1> %3, <8 x i32> %1, <8 x i32> %2 + ret <8 x i32> %4 +} + +define <8 x i16> @max_gt_v8i16c() { +; SSE-LABEL: max_gt_v8i16c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [65535,65533,65533,65535,7,5,5,7] +; SSE-NEXT: retq +; +; AVX-LABEL: max_gt_v8i16c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [65535,65533,65533,65535,7,5,5,7] +; AVX-NEXT: retq + %1 = insertelement <8 x i16> , i16 -7, i32 0 + %2 = insertelement <8 x i16> , i16 -1, i32 0 + %3 = icmp sgt <8 x i16> %1, %2 + %4 = select <8 x i1> %3, <8 x i16> %1, <8 x i16> %2 + ret <8 x i16> %4 +} + +define <16 x i16> @max_gt_v16i16c() { +; SSE-LABEL: max_gt_v16i16c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [65535,65534,65533,65532,65533,65534,65535,0] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [7,6,5,4,5,6,7,8] +; SSE-NEXT: retq +; +; AVX-LABEL: max_gt_v16i16c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [65535,65534,65533,65532,65533,65534,65535,0,7,6,5,4,5,6,7,8] +; AVX-NEXT: retq + %1 = insertelement <16 x i16> , i16 -7, i32 0 + %2 = insertelement <16 x i16> , i16 -1, i32 0 + %3 = icmp sgt <16 x i16> %1, %2 + %4 = select <16 x i1> %3, <16 x i16> %1, <16 x i16> %2 + ret <16 x i16> %4 +} + +define <16 x i8> @max_gt_v16i8c() { +; SSE-LABEL: max_gt_v16i8c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [255,254,253,252,253,254,255,0,7,6,5,4,5,6,7,8] +; SSE-NEXT: retq +; +; AVX-LABEL: max_gt_v16i8c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [255,254,253,252,253,254,255,0,7,6,5,4,5,6,7,8] +; AVX-NEXT: retq + %1 = insertelement <16 x i8> , i8 -7, i32 0 + %2 = insertelement <16 x i8> , i8 -1, i32 0 + %3 = icmp sgt <16 x i8> %1, %2 + %4 = select <16 x i1> %3, <16 x i8> %1, <16 x i8> %2 + ret <16 x i8> %4 +} + +define <2 x i64> @max_ge_v2i64c() { +; SSE-LABEL: max_ge_v2i64c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [18446744073709551615,7] +; SSE-NEXT: retq +; +; AVX-LABEL: max_ge_v2i64c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [18446744073709551615,7] +; AVX-NEXT: retq + %1 = insertelement <2 x i64> , i64 -7, i32 0 + %2 = insertelement <2 x i64> , i64 -1, i32 0 + %3 = icmp sge <2 x i64> %1, %2 + %4 = select <2 x i1> %3, <2 x i64> %1, <2 x i64> %2 + ret <2 x i64> %4 +} + +define <4 x i64> @max_ge_v4i64c() { +; SSE-LABEL: max_ge_v4i64c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm1 = [7,7] +; SSE-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: max_ge_v4i64c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [18446744073709551615,18446744073709551615,7,7] +; AVX-NEXT: retq + %1 = insertelement <4 x i64> , i64 -7, i32 0 + %2 = insertelement <4 x i64> , i64 -1, i32 0 + %3 = icmp sge <4 x i64> %1, %2 + %4 = select <4 x i1> %3, <4 x i64> %1, <4 x i64> %2 + ret <4 x i64> %4 +} + +define <4 x i32> @max_ge_v4i32c() { +; SSE-LABEL: max_ge_v4i32c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [4294967295,4294967295,7,7] +; SSE-NEXT: retq +; +; AVX-LABEL: max_ge_v4i32c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [4294967295,4294967295,7,7] +; AVX-NEXT: retq + %1 = insertelement <4 x i32> , i32 -7, i32 0 + %2 = insertelement <4 x i32> , i32 -1, i32 0 + %3 = icmp sge <4 x i32> %1, %2 + %4 = select <4 x i1> %3, <4 x i32> %1, <4 x i32> %2 + ret <4 x i32> %4 +} + +define <8 x i32> @max_ge_v8i32c() { +; SSE-LABEL: max_ge_v8i32c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [4294967295,4294967293,4294967293,4294967295] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [7,5,5,7] +; SSE-NEXT: retq +; +; AVX-LABEL: max_ge_v8i32c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [4294967295,4294967293,4294967293,4294967295,7,5,5,7] +; AVX-NEXT: retq + %1 = insertelement <8 x i32> , i32 -7, i32 0 + %2 = insertelement <8 x i32> , i32 -1, i32 0 + %3 = icmp sge <8 x i32> %1, %2 + %4 = select <8 x i1> %3, <8 x i32> %1, <8 x i32> %2 + ret <8 x i32> %4 +} + +define <8 x i16> @max_ge_v8i16c() { +; SSE-LABEL: max_ge_v8i16c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [65535,65533,65533,65535,7,5,5,7] +; SSE-NEXT: retq +; +; AVX-LABEL: max_ge_v8i16c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [65535,65533,65533,65535,7,5,5,7] +; AVX-NEXT: retq + %1 = insertelement <8 x i16> , i16 -7, i32 0 + %2 = insertelement <8 x i16> , i16 -1, i32 0 + %3 = icmp sge <8 x i16> %1, %2 + %4 = select <8 x i1> %3, <8 x i16> %1, <8 x i16> %2 + ret <8 x i16> %4 +} + +define <16 x i16> @max_ge_v16i16c() { +; SSE-LABEL: max_ge_v16i16c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [65535,65534,65533,65532,65533,65534,65535,0] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [7,6,5,4,5,6,7,8] +; SSE-NEXT: retq +; +; AVX-LABEL: max_ge_v16i16c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [65535,65534,65533,65532,65533,65534,65535,0,7,6,5,4,5,6,7,8] +; AVX-NEXT: retq + %1 = insertelement <16 x i16> , i16 -7, i32 0 + %2 = insertelement <16 x i16> , i16 -1, i32 0 + %3 = icmp sge <16 x i16> %1, %2 + %4 = select <16 x i1> %3, <16 x i16> %1, <16 x i16> %2 + ret <16 x i16> %4 +} + +define <16 x i8> @max_ge_v16i8c() { +; SSE-LABEL: max_ge_v16i8c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [255,254,253,252,253,254,255,0,7,6,5,4,5,6,7,8] +; SSE-NEXT: retq +; +; AVX-LABEL: max_ge_v16i8c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [255,254,253,252,253,254,255,0,7,6,5,4,5,6,7,8] +; AVX-NEXT: retq + %1 = insertelement <16 x i8> , i8 -7, i32 0 + %2 = insertelement <16 x i8> , i8 -1, i32 0 + %3 = icmp sge <16 x i8> %1, %2 + %4 = select <16 x i1> %3, <16 x i8> %1, <16 x i8> %2 + ret <16 x i8> %4 +} + +define <2 x i64> @min_lt_v2i64c() { +; SSE-LABEL: min_lt_v2i64c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [18446744073709551609,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_lt_v2i64c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [18446744073709551609,1] +; AVX-NEXT: retq + %1 = insertelement <2 x i64> , i64 -7, i32 0 + %2 = insertelement <2 x i64> , i64 -1, i32 0 + %3 = icmp slt <2 x i64> %1, %2 + %4 = select <2 x i1> %3, <2 x i64> %1, <2 x i64> %2 + ret <2 x i64> %4 +} + +define <4 x i64> @min_lt_v4i64c() { +; SSE-LABEL: min_lt_v4i64c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [18446744073709551609,18446744073709551609] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [1,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_lt_v4i64c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [18446744073709551609,18446744073709551609,1,1] +; AVX-NEXT: retq + %1 = insertelement <4 x i64> , i64 -7, i32 0 + %2 = insertelement <4 x i64> , i64 -1, i32 0 + %3 = icmp slt <4 x i64> %1, %2 + %4 = select <4 x i1> %3, <4 x i64> %1, <4 x i64> %2 + ret <4 x i64> %4 +} + +define <4 x i32> @min_lt_v4i32c() { +; SSE-LABEL: min_lt_v4i32c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [4294967289,4294967289,1,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_lt_v4i32c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [4294967289,4294967289,1,1] +; AVX-NEXT: retq + %1 = insertelement <4 x i32> , i32 -7, i32 0 + %2 = insertelement <4 x i32> , i32 -1, i32 0 + %3 = icmp slt <4 x i32> %1, %2 + %4 = select <4 x i1> %3, <4 x i32> %1, <4 x i32> %2 + ret <4 x i32> %4 +} + +define <8 x i32> @min_lt_v8i32c() { +; SSE-LABEL: min_lt_v8i32c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [4294967289,4294967291,4294967291,4294967289] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [1,3,3,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_lt_v8i32c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [4294967289,4294967291,4294967291,4294967289,1,3,3,1] +; AVX-NEXT: retq + %1 = insertelement <8 x i32> , i32 -7, i32 0 + %2 = insertelement <8 x i32> , i32 -1, i32 0 + %3 = icmp slt <8 x i32> %1, %2 + %4 = select <8 x i1> %3, <8 x i32> %1, <8 x i32> %2 + ret <8 x i32> %4 +} + +define <8 x i16> @min_lt_v8i16c() { +; SSE-LABEL: min_lt_v8i16c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [65529,65531,65531,65529,1,3,3,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_lt_v8i16c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [65529,65531,65531,65529,1,3,3,1] +; AVX-NEXT: retq + %1 = insertelement <8 x i16> , i16 -7, i32 0 + %2 = insertelement <8 x i16> , i16 -1, i32 0 + %3 = icmp slt <8 x i16> %1, %2 + %4 = select <8 x i1> %3, <8 x i16> %1, <8 x i16> %2 + ret <8 x i16> %4 +} + +define <16 x i16> @min_lt_v16i16c() { +; SSE-LABEL: min_lt_v16i16c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [65529,65530,65531,65532,65531,65530,65529,0] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [1,2,3,4,3,2,1,0] +; SSE-NEXT: retq +; +; AVX-LABEL: min_lt_v16i16c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [65529,65530,65531,65532,65531,65530,65529,0,1,2,3,4,3,2,1,0] +; AVX-NEXT: retq + %1 = insertelement <16 x i16> , i16 -7, i32 0 + %2 = insertelement <16 x i16> , i16 -1, i32 0 + %3 = icmp slt <16 x i16> %1, %2 + %4 = select <16 x i1> %3, <16 x i16> %1, <16 x i16> %2 + ret <16 x i16> %4 +} + +define <16 x i8> @min_lt_v16i8c() { +; SSE-LABEL: min_lt_v16i8c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [249,250,251,252,251,250,249,0,1,2,3,4,3,2,1,0] +; SSE-NEXT: retq +; +; AVX-LABEL: min_lt_v16i8c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [249,250,251,252,251,250,249,0,1,2,3,4,3,2,1,0] +; AVX-NEXT: retq + %1 = insertelement <16 x i8> , i8 -7, i32 0 + %2 = insertelement <16 x i8> , i8 -1, i32 0 + %3 = icmp slt <16 x i8> %1, %2 + %4 = select <16 x i1> %3, <16 x i8> %1, <16 x i8> %2 + ret <16 x i8> %4 +} + +define <2 x i64> @min_le_v2i64c() { +; SSE-LABEL: min_le_v2i64c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [18446744073709551609,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_le_v2i64c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [18446744073709551609,1] +; AVX-NEXT: retq + %1 = insertelement <2 x i64> , i64 -7, i32 0 + %2 = insertelement <2 x i64> , i64 -1, i32 0 + %3 = icmp sle <2 x i64> %1, %2 + %4 = select <2 x i1> %3, <2 x i64> %1, <2 x i64> %2 + ret <2 x i64> %4 +} + +define <4 x i64> @min_le_v4i64c() { +; SSE-LABEL: min_le_v4i64c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [18446744073709551609,18446744073709551609] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [1,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_le_v4i64c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [18446744073709551609,18446744073709551609,1,1] +; AVX-NEXT: retq + %1 = insertelement <4 x i64> , i64 -7, i32 0 + %2 = insertelement <4 x i64> , i64 -1, i32 0 + %3 = icmp sle <4 x i64> %1, %2 + %4 = select <4 x i1> %3, <4 x i64> %1, <4 x i64> %2 + ret <4 x i64> %4 +} + +define <4 x i32> @min_le_v4i32c() { +; SSE-LABEL: min_le_v4i32c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [4294967289,4294967289,1,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_le_v4i32c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [4294967289,4294967289,1,1] +; AVX-NEXT: retq + %1 = insertelement <4 x i32> , i32 -7, i32 0 + %2 = insertelement <4 x i32> , i32 -1, i32 0 + %3 = icmp sle <4 x i32> %1, %2 + %4 = select <4 x i1> %3, <4 x i32> %1, <4 x i32> %2 + ret <4 x i32> %4 +} + +define <8 x i32> @min_le_v8i32c() { +; SSE-LABEL: min_le_v8i32c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [4294967289,4294967291,4294967291,4294967289] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [1,3,3,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_le_v8i32c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [4294967289,4294967291,4294967291,4294967289,1,3,3,1] +; AVX-NEXT: retq + %1 = insertelement <8 x i32> , i32 -7, i32 0 + %2 = insertelement <8 x i32> , i32 -1, i32 0 + %3 = icmp sle <8 x i32> %1, %2 + %4 = select <8 x i1> %3, <8 x i32> %1, <8 x i32> %2 + ret <8 x i32> %4 +} + +define <8 x i16> @min_le_v8i16c() { +; SSE-LABEL: min_le_v8i16c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [65529,65531,65531,65529,1,3,3,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_le_v8i16c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [65529,65531,65531,65529,1,3,3,1] +; AVX-NEXT: retq + %1 = insertelement <8 x i16> , i16 -7, i32 0 + %2 = insertelement <8 x i16> , i16 -1, i32 0 + %3 = icmp sle <8 x i16> %1, %2 + %4 = select <8 x i1> %3, <8 x i16> %1, <8 x i16> %2 + ret <8 x i16> %4 +} + +define <16 x i16> @min_le_v16i16c() { +; SSE-LABEL: min_le_v16i16c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [65529,65530,65531,65532,65531,65530,65529,0] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [1,2,3,4,3,2,1,0] +; SSE-NEXT: retq +; +; AVX-LABEL: min_le_v16i16c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [65529,65530,65531,65532,65531,65530,65529,0,1,2,3,4,3,2,1,0] +; AVX-NEXT: retq + %1 = insertelement <16 x i16> , i16 -7, i32 0 + %2 = insertelement <16 x i16> , i16 -1, i32 0 + %3 = icmp sle <16 x i16> %1, %2 + %4 = select <16 x i1> %3, <16 x i16> %1, <16 x i16> %2 + ret <16 x i16> %4 +} + +define <16 x i8> @min_le_v16i8c() { +; SSE-LABEL: min_le_v16i8c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [249,250,251,252,251,250,249,0,1,2,3,4,3,2,1,0] +; SSE-NEXT: retq +; +; AVX-LABEL: min_le_v16i8c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [249,250,251,252,251,250,249,0,1,2,3,4,3,2,1,0] +; AVX-NEXT: retq + %1 = insertelement <16 x i8> , i8 -7, i32 0 + %2 = insertelement <16 x i8> , i8 -1, i32 0 + %3 = icmp sle <16 x i8> %1, %2 + %4 = select <16 x i1> %3, <16 x i8> %1, <16 x i8> %2 + ret <16 x i8> %4 +} diff --git a/test/CodeGen/X86/vec_minmax_uint.ll b/test/CodeGen/X86/vec_minmax_uint.ll new file mode 100644 index 000000000000..6e48423c1520 --- /dev/null +++ b/test/CodeGen/X86/vec_minmax_uint.ll @@ -0,0 +1,2229 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE42 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW + +; +; Unsigned Maximum (GT) +; + +define <2 x i64> @max_gt_v2i64(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: max_gt_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_gt_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm1, %xmm3 +; SSE41-NEXT: pxor %xmm0, %xmm3 +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm5, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: por %xmm3, %xmm0 +; SSE41-NEXT: blendvpd %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_gt_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa %xmm0, %xmm2 +; SSE42-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE42-NEXT: movdqa %xmm1, %xmm3 +; SSE42-NEXT: pxor %xmm0, %xmm3 +; SSE42-NEXT: pxor %xmm2, %xmm0 +; SSE42-NEXT: pcmpgtq %xmm3, %xmm0 +; SSE42-NEXT: blendvpd %xmm2, %xmm1 +; SSE42-NEXT: movapd %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: max_gt_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX-NEXT: vpxor %xmm2, %xmm1, %xmm3 +; AVX-NEXT: vpxor %xmm2, %xmm0, %xmm2 +; AVX-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq + %1 = icmp ugt <2 x i64> %a, %b + %2 = select <2 x i1> %1, <2 x i64> %a, <2 x i64> %b + ret <2 x i64> %2 +} + +define <4 x i64> @max_gt_v4i64(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: max_gt_v4i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm1, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: movdqa %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm5, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm2, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: pxor %xmm0, %xmm4 +; SSE2-NEXT: movdqa %xmm4, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm5, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: pand %xmm6, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm6 +; SSE2-NEXT: por %xmm6, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_gt_v4i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm8 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm3, %xmm5 +; SSE41-NEXT: pxor %xmm0, %xmm5 +; SSE41-NEXT: movdqa %xmm1, %xmm6 +; SSE41-NEXT: pxor %xmm0, %xmm6 +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] +; SSE41-NEXT: pand %xmm4, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE41-NEXT: por %xmm6, %xmm5 +; SSE41-NEXT: movdqa %xmm2, %xmm4 +; SSE41-NEXT: pxor %xmm0, %xmm4 +; SSE41-NEXT: pxor %xmm8, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm7, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] +; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: blendvpd %xmm8, %xmm2 +; SSE41-NEXT: movdqa %xmm5, %xmm0 +; SSE41-NEXT: blendvpd %xmm1, %xmm3 +; SSE41-NEXT: movapd %xmm2, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_gt_v4i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa %xmm0, %xmm4 +; SSE42-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE42-NEXT: movdqa %xmm3, %xmm6 +; SSE42-NEXT: pxor %xmm0, %xmm6 +; SSE42-NEXT: movdqa %xmm1, %xmm5 +; SSE42-NEXT: pxor %xmm0, %xmm5 +; SSE42-NEXT: pcmpgtq %xmm6, %xmm5 +; SSE42-NEXT: movdqa %xmm2, %xmm6 +; SSE42-NEXT: pxor %xmm0, %xmm6 +; SSE42-NEXT: pxor %xmm4, %xmm0 +; SSE42-NEXT: pcmpgtq %xmm6, %xmm0 +; SSE42-NEXT: blendvpd %xmm4, %xmm2 +; SSE42-NEXT: movdqa %xmm5, %xmm0 +; SSE42-NEXT: blendvpd %xmm1, %xmm3 +; SSE42-NEXT: movapd %xmm2, %xmm0 +; SSE42-NEXT: movapd %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: max_gt_v4i64: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm4, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vxorps %xmm3, %xmm1, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm0, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: max_gt_v4i64: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpxor %ymm2, %ymm1, %ymm3 +; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm2 +; AVX2-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: max_gt_v4i64: +; AVX512: # BB#0: +; AVX512-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX512-NEXT: vpxor %ymm2, %ymm1, %ymm3 +; AVX512-NEXT: vpxor %ymm2, %ymm0, %ymm2 +; AVX512-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2 +; AVX512-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX512-NEXT: retq + %1 = icmp ugt <4 x i64> %a, %b + %2 = select <4 x i1> %1, <4 x i64> %a, <4 x i64> %b + ret <4 x i64> %2 +} + +define <4 x i32> @max_gt_v4i32(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: max_gt_v4i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_gt_v4i32: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxud %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_gt_v4i32: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxud %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: max_gt_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpmaxud %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp ugt <4 x i32> %a, %b + %2 = select <4 x i1> %1, <4 x i32> %a, <4 x i32> %b + ret <4 x i32> %2 +} + +define <8 x i32> @max_gt_v8i32(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: max_gt_v8i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pxor %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm2, %xmm6 +; SSE2-NEXT: pxor %xmm5, %xmm6 +; SSE2-NEXT: pxor %xmm0, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm1, %xmm4 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_gt_v8i32: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxud %xmm2, %xmm0 +; SSE41-NEXT: pmaxud %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_gt_v8i32: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxud %xmm2, %xmm0 +; SSE42-NEXT: pmaxud %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: max_gt_v8i32: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxud %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxud %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: max_gt_v8i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpmaxud %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: max_gt_v8i32: +; AVX512: # BB#0: +; AVX512-NEXT: vpmaxud %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp ugt <8 x i32> %a, %b + %2 = select <8 x i1> %1, <8 x i32> %a, <8 x i32> %b + ret <8 x i32> %2 +} + +define <8 x i16> @max_gt_v8i16(<8 x i16> %a, <8 x i16> %b) { +; SSE2-LABEL: max_gt_v8i16: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtw %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_gt_v8i16: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxuw %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_gt_v8i16: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxuw %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: max_gt_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpmaxuw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp ugt <8 x i16> %a, %b + %2 = select <8 x i1> %1, <8 x i16> %a, <8 x i16> %b + ret <8 x i16> %2 +} + +define <16 x i16> @max_gt_v16i16(<16 x i16> %a, <16 x i16> %b) { +; SSE2-LABEL: max_gt_v16i16: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pxor %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: pcmpgtw %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm2, %xmm6 +; SSE2-NEXT: pxor %xmm5, %xmm6 +; SSE2-NEXT: pxor %xmm0, %xmm5 +; SSE2-NEXT: pcmpgtw %xmm6, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm1, %xmm4 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_gt_v16i16: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxuw %xmm2, %xmm0 +; SSE41-NEXT: pmaxuw %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_gt_v16i16: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxuw %xmm2, %xmm0 +; SSE42-NEXT: pmaxuw %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: max_gt_v16i16: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxuw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxuw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: max_gt_v16i16: +; AVX2: # BB#0: +; AVX2-NEXT: vpmaxuw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: max_gt_v16i16: +; AVX512: # BB#0: +; AVX512-NEXT: vpmaxuw %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp ugt <16 x i16> %a, %b + %2 = select <16 x i1> %1, <16 x i16> %a, <16 x i16> %b + ret <16 x i16> %2 +} + +define <16 x i8> @max_gt_v16i8(<16 x i8> %a, <16 x i8> %b) { +; SSE-LABEL: max_gt_v16i8: +; SSE: # BB#0: +; SSE-NEXT: pmaxub %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: max_gt_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpmaxub %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp ugt <16 x i8> %a, %b + %2 = select <16 x i1> %1, <16 x i8> %a, <16 x i8> %b + ret <16 x i8> %2 +} + +define <32 x i8> @max_gt_v32i8(<32 x i8> %a, <32 x i8> %b) { +; SSE-LABEL: max_gt_v32i8: +; SSE: # BB#0: +; SSE-NEXT: pmaxub %xmm2, %xmm0 +; SSE-NEXT: pmaxub %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: max_gt_v32i8: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxub %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxub %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: max_gt_v32i8: +; AVX2: # BB#0: +; AVX2-NEXT: vpmaxub %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: max_gt_v32i8: +; AVX512: # BB#0: +; AVX512-NEXT: vpmaxub %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp ugt <32 x i8> %a, %b + %2 = select <32 x i1> %1, <32 x i8> %a, <32 x i8> %b + ret <32 x i8> %2 +} + +; +; Unsigned Maximum (GE) +; + +define <2 x i64> @max_ge_v2i64(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: max_ge_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_ge_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm2, %xmm3 +; SSE41-NEXT: pxor %xmm0, %xmm3 +; SSE41-NEXT: pxor %xmm1, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE41-NEXT: por %xmm0, %xmm3 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: blendvpd %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_ge_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa %xmm0, %xmm2 +; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; SSE42-NEXT: pxor %xmm3, %xmm0 +; SSE42-NEXT: pxor %xmm1, %xmm3 +; SSE42-NEXT: pcmpgtq %xmm0, %xmm3 +; SSE42-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE42-NEXT: pxor %xmm3, %xmm0 +; SSE42-NEXT: blendvpd %xmm2, %xmm1 +; SSE42-NEXT: movapd %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: max_ge_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX-NEXT: vpxor %xmm2, %xmm0, %xmm3 +; AVX-NEXT: vpxor %xmm2, %xmm1, %xmm2 +; AVX-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq + %1 = icmp uge <2 x i64> %a, %b + %2 = select <2 x i1> %1, <2 x i64> %a, <2 x i64> %b + ret <2 x i64> %2 +} + +define <4 x i64> @max_ge_v4i64(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: max_ge_v4i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm3, %xmm5 +; SSE2-NEXT: pxor %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm8 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm8, %xmm9 +; SSE2-NEXT: pxor %xmm4, %xmm9 +; SSE2-NEXT: movdqa %xmm0, %xmm6 +; SSE2-NEXT: pxor %xmm7, %xmm6 +; SSE2-NEXT: pxor %xmm2, %xmm7 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm5[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: pand %xmm10, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm1, %xmm8 +; SSE2-NEXT: pandn %xmm3, %xmm9 +; SSE2-NEXT: por %xmm8, %xmm9 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm9, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_ge_v4i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm8 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm1, %xmm5 +; SSE41-NEXT: pxor %xmm0, %xmm5 +; SSE41-NEXT: movdqa %xmm3, %xmm6 +; SSE41-NEXT: pxor %xmm0, %xmm6 +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] +; SSE41-NEXT: pand %xmm4, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE41-NEXT: por %xmm6, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm9 +; SSE41-NEXT: pxor %xmm9, %xmm5 +; SSE41-NEXT: movdqa %xmm8, %xmm6 +; SSE41-NEXT: pxor %xmm0, %xmm6 +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm4, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: pxor %xmm9, %xmm0 +; SSE41-NEXT: blendvpd %xmm8, %xmm2 +; SSE41-NEXT: movdqa %xmm5, %xmm0 +; SSE41-NEXT: blendvpd %xmm1, %xmm3 +; SSE41-NEXT: movapd %xmm2, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_ge_v4i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa %xmm0, %xmm4 +; SSE42-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE42-NEXT: movdqa %xmm1, %xmm6 +; SSE42-NEXT: pxor %xmm0, %xmm6 +; SSE42-NEXT: movdqa %xmm3, %xmm5 +; SSE42-NEXT: pxor %xmm0, %xmm5 +; SSE42-NEXT: pcmpgtq %xmm6, %xmm5 +; SSE42-NEXT: pcmpeqd %xmm6, %xmm6 +; SSE42-NEXT: pxor %xmm6, %xmm5 +; SSE42-NEXT: movdqa %xmm4, %xmm7 +; SSE42-NEXT: pxor %xmm0, %xmm7 +; SSE42-NEXT: pxor %xmm2, %xmm0 +; SSE42-NEXT: pcmpgtq %xmm7, %xmm0 +; SSE42-NEXT: pxor %xmm6, %xmm0 +; SSE42-NEXT: blendvpd %xmm4, %xmm2 +; SSE42-NEXT: movdqa %xmm5, %xmm0 +; SSE42-NEXT: blendvpd %xmm1, %xmm3 +; SSE42-NEXT: movapd %xmm2, %xmm0 +; SSE42-NEXT: movapd %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: max_ge_v4i64: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm4, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm4, %xmm4, %xmm4 +; AVX1-NEXT: vpxor %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vxorps %xmm3, %xmm0, %xmm5 +; AVX1-NEXT: vxorps %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm5, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: max_ge_v4i64: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm3 +; AVX2-NEXT: vpxor %ymm2, %ymm1, %ymm2 +; AVX2-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3 +; AVX2-NEXT: vpxor %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: max_ge_v4i64: +; AVX512: # BB#0: +; AVX512-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX512-NEXT: vpxor %ymm2, %ymm0, %ymm3 +; AVX512-NEXT: vpxor %ymm2, %ymm1, %ymm2 +; AVX512-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2 +; AVX512-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3 +; AVX512-NEXT: vpxor %ymm3, %ymm2, %ymm2 +; AVX512-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX512-NEXT: retq + %1 = icmp uge <4 x i64> %a, %b + %2 = select <4 x i1> %1, <4 x i64> %a, <4 x i64> %b + ret <4 x i64> %2 +} + +define <4 x i32> @max_ge_v4i32(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: max_ge_v4i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pxor %xmm1, %xmm3 +; SSE2-NEXT: pcmpgtd %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_ge_v4i32: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxud %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_ge_v4i32: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxud %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: max_ge_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpmaxud %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp uge <4 x i32> %a, %b + %2 = select <4 x i1> %1, <4 x i32> %a, <4 x i32> %b + ret <4 x i32> %2 +} + +define <8 x i32> @max_ge_v8i32(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: max_ge_v8i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm3, %xmm7 +; SSE2-NEXT: pxor %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm7 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm0, %xmm8 +; SSE2-NEXT: pxor %xmm6, %xmm8 +; SSE2-NEXT: pxor %xmm2, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm6 +; SSE2-NEXT: pxor %xmm6, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm6 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm6, %xmm4 +; SSE2-NEXT: pandn %xmm1, %xmm7 +; SSE2-NEXT: pandn %xmm3, %xmm5 +; SSE2-NEXT: por %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_ge_v8i32: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxud %xmm2, %xmm0 +; SSE41-NEXT: pmaxud %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_ge_v8i32: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxud %xmm2, %xmm0 +; SSE42-NEXT: pmaxud %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: max_ge_v8i32: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxud %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxud %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: max_ge_v8i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpmaxud %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: max_ge_v8i32: +; AVX512: # BB#0: +; AVX512-NEXT: vpmaxud %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp uge <8 x i32> %a, %b + %2 = select <8 x i1> %1, <8 x i32> %a, <8 x i32> %b + ret <8 x i32> %2 +} + +define <8 x i16> @max_ge_v8i16(<8 x i16> %a, <8 x i16> %b) { +; SSE2-LABEL: max_ge_v8i16: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: psubusw %xmm0, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm3 +; SSE2-NEXT: pcmpeqw %xmm2, %xmm3 +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_ge_v8i16: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxuw %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_ge_v8i16: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxuw %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: max_ge_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpmaxuw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp uge <8 x i16> %a, %b + %2 = select <8 x i1> %1, <8 x i16> %a, <8 x i16> %b + ret <8 x i16> %2 +} + +define <16 x i16> @max_ge_v16i16(<16 x i16> %a, <16 x i16> %b) { +; SSE2-LABEL: max_ge_v16i16: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: psubusw %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm5, %xmm5 +; SSE2-NEXT: pcmpeqw %xmm5, %xmm4 +; SSE2-NEXT: movdqa %xmm2, %xmm6 +; SSE2-NEXT: psubusw %xmm0, %xmm6 +; SSE2-NEXT: pcmpeqw %xmm5, %xmm6 +; SSE2-NEXT: pand %xmm6, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm6 +; SSE2-NEXT: por %xmm6, %xmm0 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: max_ge_v16i16: +; SSE41: # BB#0: +; SSE41-NEXT: pmaxuw %xmm2, %xmm0 +; SSE41-NEXT: pmaxuw %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: max_ge_v16i16: +; SSE42: # BB#0: +; SSE42-NEXT: pmaxuw %xmm2, %xmm0 +; SSE42-NEXT: pmaxuw %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: max_ge_v16i16: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxuw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxuw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: max_ge_v16i16: +; AVX2: # BB#0: +; AVX2-NEXT: vpmaxuw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: max_ge_v16i16: +; AVX512: # BB#0: +; AVX512-NEXT: vpmaxuw %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp uge <16 x i16> %a, %b + %2 = select <16 x i1> %1, <16 x i16> %a, <16 x i16> %b + ret <16 x i16> %2 +} + +define <16 x i8> @max_ge_v16i8(<16 x i8> %a, <16 x i8> %b) { +; SSE-LABEL: max_ge_v16i8: +; SSE: # BB#0: +; SSE-NEXT: pmaxub %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: max_ge_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpmaxub %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp uge <16 x i8> %a, %b + %2 = select <16 x i1> %1, <16 x i8> %a, <16 x i8> %b + ret <16 x i8> %2 +} + +define <32 x i8> @max_ge_v32i8(<32 x i8> %a, <32 x i8> %b) { +; SSE-LABEL: max_ge_v32i8: +; SSE: # BB#0: +; SSE-NEXT: pmaxub %xmm2, %xmm0 +; SSE-NEXT: pmaxub %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: max_ge_v32i8: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxub %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxub %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: max_ge_v32i8: +; AVX2: # BB#0: +; AVX2-NEXT: vpmaxub %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: max_ge_v32i8: +; AVX512: # BB#0: +; AVX512-NEXT: vpmaxub %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp uge <32 x i8> %a, %b + %2 = select <32 x i1> %1, <32 x i8> %a, <32 x i8> %b + ret <32 x i8> %2 +} + +; +; Unsigned Minimum (LT) +; + +define <2 x i64> @min_lt_v2i64(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: min_lt_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_lt_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm2, %xmm3 +; SSE41-NEXT: pxor %xmm0, %xmm3 +; SSE41-NEXT: pxor %xmm1, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm5, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3] +; SSE41-NEXT: por %xmm3, %xmm0 +; SSE41-NEXT: blendvpd %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_lt_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa %xmm0, %xmm2 +; SSE42-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE42-NEXT: movdqa %xmm2, %xmm3 +; SSE42-NEXT: pxor %xmm0, %xmm3 +; SSE42-NEXT: pxor %xmm1, %xmm0 +; SSE42-NEXT: pcmpgtq %xmm3, %xmm0 +; SSE42-NEXT: blendvpd %xmm2, %xmm1 +; SSE42-NEXT: movapd %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: min_lt_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX-NEXT: vpxor %xmm2, %xmm0, %xmm3 +; AVX-NEXT: vpxor %xmm2, %xmm1, %xmm2 +; AVX-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq + %1 = icmp ult <2 x i64> %a, %b + %2 = select <2 x i1> %1, <2 x i64> %a, <2 x i64> %b + ret <2 x i64> %2 +} + +define <4 x i64> @min_lt_v4i64(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: min_lt_v4i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: movdqa %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm5, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: pxor %xmm2, %xmm4 +; SSE2-NEXT: movdqa %xmm4, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm5, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: pand %xmm6, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm6 +; SSE2-NEXT: por %xmm6, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_lt_v4i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm8 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm1, %xmm5 +; SSE41-NEXT: pxor %xmm0, %xmm5 +; SSE41-NEXT: movdqa %xmm3, %xmm6 +; SSE41-NEXT: pxor %xmm0, %xmm6 +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] +; SSE41-NEXT: pand %xmm4, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE41-NEXT: por %xmm6, %xmm5 +; SSE41-NEXT: movdqa %xmm8, %xmm4 +; SSE41-NEXT: pxor %xmm0, %xmm4 +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm6 +; SSE41-NEXT: pcmpgtd %xmm4, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm7, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3] +; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: blendvpd %xmm8, %xmm2 +; SSE41-NEXT: movdqa %xmm5, %xmm0 +; SSE41-NEXT: blendvpd %xmm1, %xmm3 +; SSE41-NEXT: movapd %xmm2, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_lt_v4i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa %xmm0, %xmm4 +; SSE42-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE42-NEXT: movdqa %xmm1, %xmm6 +; SSE42-NEXT: pxor %xmm0, %xmm6 +; SSE42-NEXT: movdqa %xmm3, %xmm5 +; SSE42-NEXT: pxor %xmm0, %xmm5 +; SSE42-NEXT: pcmpgtq %xmm6, %xmm5 +; SSE42-NEXT: movdqa %xmm4, %xmm6 +; SSE42-NEXT: pxor %xmm0, %xmm6 +; SSE42-NEXT: pxor %xmm2, %xmm0 +; SSE42-NEXT: pcmpgtq %xmm6, %xmm0 +; SSE42-NEXT: blendvpd %xmm4, %xmm2 +; SSE42-NEXT: movdqa %xmm5, %xmm0 +; SSE42-NEXT: blendvpd %xmm1, %xmm3 +; SSE42-NEXT: movapd %xmm2, %xmm0 +; SSE42-NEXT: movapd %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: min_lt_v4i64: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm4, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vxorps %xmm3, %xmm0, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: min_lt_v4i64: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm3 +; AVX2-NEXT: vpxor %ymm2, %ymm1, %ymm2 +; AVX2-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: min_lt_v4i64: +; AVX512: # BB#0: +; AVX512-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX512-NEXT: vpxor %ymm2, %ymm0, %ymm3 +; AVX512-NEXT: vpxor %ymm2, %ymm1, %ymm2 +; AVX512-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2 +; AVX512-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX512-NEXT: retq + %1 = icmp ult <4 x i64> %a, %b + %2 = select <4 x i1> %1, <4 x i64> %a, <4 x i64> %b + ret <4 x i64> %2 +} + +define <4 x i32> @min_lt_v4i32(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: min_lt_v4i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_lt_v4i32: +; SSE41: # BB#0: +; SSE41-NEXT: pminud %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_lt_v4i32: +; SSE42: # BB#0: +; SSE42-NEXT: pminud %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: min_lt_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpminud %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp ult <4 x i32> %a, %b + %2 = select <4 x i1> %1, <4 x i32> %a, <4 x i32> %b + ret <4 x i32> %2 +} + +define <8 x i32> @min_lt_v8i32(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: min_lt_v8i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: pxor %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm4 +; SSE2-NEXT: pand %xmm4, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm4, %xmm0 +; SSE2-NEXT: pand %xmm6, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm6 +; SSE2-NEXT: por %xmm6, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_lt_v8i32: +; SSE41: # BB#0: +; SSE41-NEXT: pminud %xmm2, %xmm0 +; SSE41-NEXT: pminud %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_lt_v8i32: +; SSE42: # BB#0: +; SSE42-NEXT: pminud %xmm2, %xmm0 +; SSE42-NEXT: pminud %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: min_lt_v8i32: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminud %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminud %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: min_lt_v8i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpminud %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: min_lt_v8i32: +; AVX512: # BB#0: +; AVX512-NEXT: vpminud %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp ult <8 x i32> %a, %b + %2 = select <8 x i1> %1, <8 x i32> %a, <8 x i32> %b + ret <8 x i32> %2 +} + +define <8 x i16> @min_lt_v8i16(<8 x i16> %a, <8 x i16> %b) { +; SSE2-LABEL: min_lt_v8i16: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtw %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_lt_v8i16: +; SSE41: # BB#0: +; SSE41-NEXT: pminuw %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_lt_v8i16: +; SSE42: # BB#0: +; SSE42-NEXT: pminuw %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: min_lt_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpminuw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp ult <8 x i16> %a, %b + %2 = select <8 x i1> %1, <8 x i16> %a, <8 x i16> %b + ret <8 x i16> %2 +} + +define <16 x i16> @min_lt_v16i16(<16 x i16> %a, <16 x i16> %b) { +; SSE2-LABEL: min_lt_v16i16: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: pcmpgtw %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: pxor %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtw %xmm5, %xmm4 +; SSE2-NEXT: pand %xmm4, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm4, %xmm0 +; SSE2-NEXT: pand %xmm6, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm6 +; SSE2-NEXT: por %xmm6, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_lt_v16i16: +; SSE41: # BB#0: +; SSE41-NEXT: pminuw %xmm2, %xmm0 +; SSE41-NEXT: pminuw %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_lt_v16i16: +; SSE42: # BB#0: +; SSE42-NEXT: pminuw %xmm2, %xmm0 +; SSE42-NEXT: pminuw %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: min_lt_v16i16: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminuw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminuw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: min_lt_v16i16: +; AVX2: # BB#0: +; AVX2-NEXT: vpminuw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: min_lt_v16i16: +; AVX512: # BB#0: +; AVX512-NEXT: vpminuw %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp ult <16 x i16> %a, %b + %2 = select <16 x i1> %1, <16 x i16> %a, <16 x i16> %b + ret <16 x i16> %2 +} + +define <16 x i8> @min_lt_v16i8(<16 x i8> %a, <16 x i8> %b) { +; SSE-LABEL: min_lt_v16i8: +; SSE: # BB#0: +; SSE-NEXT: pminub %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: min_lt_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpminub %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp ult <16 x i8> %a, %b + %2 = select <16 x i1> %1, <16 x i8> %a, <16 x i8> %b + ret <16 x i8> %2 +} + +define <32 x i8> @min_lt_v32i8(<32 x i8> %a, <32 x i8> %b) { +; SSE-LABEL: min_lt_v32i8: +; SSE: # BB#0: +; SSE-NEXT: pminub %xmm2, %xmm0 +; SSE-NEXT: pminub %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: min_lt_v32i8: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminub %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminub %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: min_lt_v32i8: +; AVX2: # BB#0: +; AVX2-NEXT: vpminub %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: min_lt_v32i8: +; AVX512: # BB#0: +; AVX512-NEXT: vpminub %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp ult <32 x i8> %a, %b + %2 = select <32 x i1> %1, <32 x i8> %a, <32 x i8> %b + ret <32 x i8> %2 +} + +; +; Unsigned Minimum (LE) +; + +define <2 x i64> @min_le_v2i64(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: min_le_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_le_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm1, %xmm3 +; SSE41-NEXT: pxor %xmm0, %xmm3 +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm4 +; SSE41-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm5, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE41-NEXT: por %xmm0, %xmm3 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE41-NEXT: pxor %xmm3, %xmm0 +; SSE41-NEXT: blendvpd %xmm2, %xmm1 +; SSE41-NEXT: movapd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_le_v2i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa %xmm0, %xmm2 +; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; SSE42-NEXT: movdqa %xmm1, %xmm0 +; SSE42-NEXT: pxor %xmm3, %xmm0 +; SSE42-NEXT: pxor %xmm2, %xmm3 +; SSE42-NEXT: pcmpgtq %xmm0, %xmm3 +; SSE42-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE42-NEXT: pxor %xmm3, %xmm0 +; SSE42-NEXT: blendvpd %xmm2, %xmm1 +; SSE42-NEXT: movapd %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: min_le_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX-NEXT: vpxor %xmm2, %xmm1, %xmm3 +; AVX-NEXT: vpxor %xmm2, %xmm0, %xmm2 +; AVX-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq + %1 = icmp ule <2 x i64> %a, %b + %2 = select <2 x i1> %1, <2 x i64> %a, <2 x i64> %b + ret <2 x i64> %2 +} + +define <4 x i64> @min_le_v4i64(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: min_le_v4i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pxor %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm8 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm8, %xmm9 +; SSE2-NEXT: pxor %xmm4, %xmm9 +; SSE2-NEXT: movdqa %xmm2, %xmm6 +; SSE2-NEXT: pxor %xmm7, %xmm6 +; SSE2-NEXT: pxor %xmm0, %xmm7 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm5[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: pand %xmm10, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm1, %xmm8 +; SSE2-NEXT: pandn %xmm3, %xmm9 +; SSE2-NEXT: por %xmm8, %xmm9 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm9, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_le_v4i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm8 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648] +; SSE41-NEXT: movdqa %xmm3, %xmm5 +; SSE41-NEXT: pxor %xmm0, %xmm5 +; SSE41-NEXT: movdqa %xmm1, %xmm6 +; SSE41-NEXT: pxor %xmm0, %xmm6 +; SSE41-NEXT: movdqa %xmm6, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm5, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] +; SSE41-NEXT: pand %xmm4, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE41-NEXT: por %xmm6, %xmm5 +; SSE41-NEXT: pcmpeqd %xmm9, %xmm9 +; SSE41-NEXT: pxor %xmm9, %xmm5 +; SSE41-NEXT: movdqa %xmm2, %xmm6 +; SSE41-NEXT: pxor %xmm0, %xmm6 +; SSE41-NEXT: pxor %xmm8, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm7 +; SSE41-NEXT: pcmpgtd %xmm6, %xmm7 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2] +; SSE41-NEXT: pcmpeqd %xmm6, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3] +; SSE41-NEXT: pand %xmm4, %xmm6 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3] +; SSE41-NEXT: por %xmm6, %xmm0 +; SSE41-NEXT: pxor %xmm9, %xmm0 +; SSE41-NEXT: blendvpd %xmm8, %xmm2 +; SSE41-NEXT: movdqa %xmm5, %xmm0 +; SSE41-NEXT: blendvpd %xmm1, %xmm3 +; SSE41-NEXT: movapd %xmm2, %xmm0 +; SSE41-NEXT: movapd %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_le_v4i64: +; SSE42: # BB#0: +; SSE42-NEXT: movdqa %xmm0, %xmm4 +; SSE42-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE42-NEXT: movdqa %xmm3, %xmm6 +; SSE42-NEXT: pxor %xmm0, %xmm6 +; SSE42-NEXT: movdqa %xmm1, %xmm5 +; SSE42-NEXT: pxor %xmm0, %xmm5 +; SSE42-NEXT: pcmpgtq %xmm6, %xmm5 +; SSE42-NEXT: pcmpeqd %xmm6, %xmm6 +; SSE42-NEXT: pxor %xmm6, %xmm5 +; SSE42-NEXT: movdqa %xmm2, %xmm7 +; SSE42-NEXT: pxor %xmm0, %xmm7 +; SSE42-NEXT: pxor %xmm4, %xmm0 +; SSE42-NEXT: pcmpgtq %xmm7, %xmm0 +; SSE42-NEXT: pxor %xmm6, %xmm0 +; SSE42-NEXT: blendvpd %xmm4, %xmm2 +; SSE42-NEXT: movdqa %xmm5, %xmm0 +; SSE42-NEXT: blendvpd %xmm1, %xmm3 +; SSE42-NEXT: movapd %xmm2, %xmm0 +; SSE42-NEXT: movapd %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: min_le_v4i64: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm4, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm4, %xmm4, %xmm4 +; AVX1-NEXT: vpxor %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vxorps %xmm3, %xmm1, %xmm5 +; AVX1-NEXT: vxorps %xmm3, %xmm0, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm5, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: min_le_v4i64: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpxor %ymm2, %ymm1, %ymm3 +; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm2 +; AVX2-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3 +; AVX2-NEXT: vpxor %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: min_le_v4i64: +; AVX512: # BB#0: +; AVX512-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX512-NEXT: vpxor %ymm2, %ymm1, %ymm3 +; AVX512-NEXT: vpxor %ymm2, %ymm0, %ymm2 +; AVX512-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2 +; AVX512-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3 +; AVX512-NEXT: vpxor %ymm3, %ymm2, %ymm2 +; AVX512-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX512-NEXT: retq + %1 = icmp ule <4 x i64> %a, %b + %2 = select <4 x i1> %1, <4 x i64> %a, <4 x i64> %b + ret <4 x i64> %2 +} + +define <4 x i32> @min_le_v4i32(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: min_le_v4i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pxor %xmm0, %xmm3 +; SSE2-NEXT: pcmpgtd %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_le_v4i32: +; SSE41: # BB#0: +; SSE41-NEXT: pminud %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_le_v4i32: +; SSE42: # BB#0: +; SSE42-NEXT: pminud %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: min_le_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpminud %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp ule <4 x i32> %a, %b + %2 = select <4 x i1> %1, <4 x i32> %a, <4 x i32> %b + ret <4 x i32> %2 +} + +define <8 x i32> @min_le_v8i32(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: min_le_v8i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pxor %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm1, %xmm7 +; SSE2-NEXT: pxor %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm7 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm2, %xmm8 +; SSE2-NEXT: pxor %xmm6, %xmm8 +; SSE2-NEXT: pxor %xmm0, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm6 +; SSE2-NEXT: pxor %xmm6, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm6 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm6, %xmm4 +; SSE2-NEXT: pandn %xmm1, %xmm7 +; SSE2-NEXT: pandn %xmm3, %xmm5 +; SSE2-NEXT: por %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_le_v8i32: +; SSE41: # BB#0: +; SSE41-NEXT: pminud %xmm2, %xmm0 +; SSE41-NEXT: pminud %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_le_v8i32: +; SSE42: # BB#0: +; SSE42-NEXT: pminud %xmm2, %xmm0 +; SSE42-NEXT: pminud %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: min_le_v8i32: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminud %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminud %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: min_le_v8i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpminud %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: min_le_v8i32: +; AVX512: # BB#0: +; AVX512-NEXT: vpminud %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp ule <8 x i32> %a, %b + %2 = select <8 x i1> %1, <8 x i32> %a, <8 x i32> %b + ret <8 x i32> %2 +} + +define <8 x i16> @min_le_v8i16(<8 x i16> %a, <8 x i16> %b) { +; SSE2-LABEL: min_le_v8i16: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: psubusw %xmm1, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm3 +; SSE2-NEXT: pcmpeqw %xmm2, %xmm3 +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_le_v8i16: +; SSE41: # BB#0: +; SSE41-NEXT: pminuw %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_le_v8i16: +; SSE42: # BB#0: +; SSE42-NEXT: pminuw %xmm1, %xmm0 +; SSE42-NEXT: retq +; +; AVX-LABEL: min_le_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpminuw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp ule <8 x i16> %a, %b + %2 = select <8 x i1> %1, <8 x i16> %a, <8 x i16> %b + ret <8 x i16> %2 +} + +define <16 x i16> @min_le_v16i16(<16 x i16> %a, <16 x i16> %b) { +; SSE2-LABEL: min_le_v16i16: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: psubusw %xmm3, %xmm4 +; SSE2-NEXT: pxor %xmm6, %xmm6 +; SSE2-NEXT: pcmpeqw %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: psubusw %xmm2, %xmm5 +; SSE2-NEXT: pcmpeqw %xmm6, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm0, %xmm5 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm1, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE41-LABEL: min_le_v16i16: +; SSE41: # BB#0: +; SSE41-NEXT: pminuw %xmm2, %xmm0 +; SSE41-NEXT: pminuw %xmm3, %xmm1 +; SSE41-NEXT: retq +; +; SSE42-LABEL: min_le_v16i16: +; SSE42: # BB#0: +; SSE42-NEXT: pminuw %xmm2, %xmm0 +; SSE42-NEXT: pminuw %xmm3, %xmm1 +; SSE42-NEXT: retq +; +; AVX1-LABEL: min_le_v16i16: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminuw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminuw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: min_le_v16i16: +; AVX2: # BB#0: +; AVX2-NEXT: vpminuw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: min_le_v16i16: +; AVX512: # BB#0: +; AVX512-NEXT: vpminuw %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp ule <16 x i16> %a, %b + %2 = select <16 x i1> %1, <16 x i16> %a, <16 x i16> %b + ret <16 x i16> %2 +} + +define <16 x i8> @min_le_v16i8(<16 x i8> %a, <16 x i8> %b) { +; SSE-LABEL: min_le_v16i8: +; SSE: # BB#0: +; SSE-NEXT: pminub %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: min_le_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpminub %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %1 = icmp ule <16 x i8> %a, %b + %2 = select <16 x i1> %1, <16 x i8> %a, <16 x i8> %b + ret <16 x i8> %2 +} + +define <32 x i8> @min_le_v32i8(<32 x i8> %a, <32 x i8> %b) { +; SSE-LABEL: min_le_v32i8: +; SSE: # BB#0: +; SSE-NEXT: pminub %xmm2, %xmm0 +; SSE-NEXT: pminub %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: min_le_v32i8: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminub %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminub %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: min_le_v32i8: +; AVX2: # BB#0: +; AVX2-NEXT: vpminub %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512-LABEL: min_le_v32i8: +; AVX512: # BB#0: +; AVX512-NEXT: vpminub %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq + %1 = icmp ule <32 x i8> %a, %b + %2 = select <32 x i1> %1, <32 x i8> %a, <32 x i8> %b + ret <32 x i8> %2 +} + +; +; Constant Folding +; + +define <2 x i64> @max_gt_v2i64c() { +; SSE-LABEL: max_gt_v2i64c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [18446744073709551615,7] +; SSE-NEXT: retq +; +; AVX-LABEL: max_gt_v2i64c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [18446744073709551615,7] +; AVX-NEXT: retq + %1 = insertelement <2 x i64> , i64 -7, i32 0 + %2 = insertelement <2 x i64> , i64 -1, i32 0 + %3 = icmp ugt <2 x i64> %1, %2 + %4 = select <2 x i1> %3, <2 x i64> %1, <2 x i64> %2 + ret <2 x i64> %4 +} + +define <4 x i64> @max_gt_v4i64c() { +; SSE-LABEL: max_gt_v4i64c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm1 = [7,7] +; SSE-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: max_gt_v4i64c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [18446744073709551615,18446744073709551615,7,7] +; AVX-NEXT: retq + %1 = insertelement <4 x i64> , i64 -7, i32 0 + %2 = insertelement <4 x i64> , i64 -1, i32 0 + %3 = icmp ugt <4 x i64> %1, %2 + %4 = select <4 x i1> %3, <4 x i64> %1, <4 x i64> %2 + ret <4 x i64> %4 +} + +define <4 x i32> @max_gt_v4i32c() { +; SSE-LABEL: max_gt_v4i32c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [4294967295,4294967295,7,7] +; SSE-NEXT: retq +; +; AVX-LABEL: max_gt_v4i32c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [4294967295,4294967295,7,7] +; AVX-NEXT: retq + %1 = insertelement <4 x i32> , i32 -7, i32 0 + %2 = insertelement <4 x i32> , i32 -1, i32 0 + %3 = icmp ugt <4 x i32> %1, %2 + %4 = select <4 x i1> %3, <4 x i32> %1, <4 x i32> %2 + ret <4 x i32> %4 +} + +define <8 x i32> @max_gt_v8i32c() { +; SSE-LABEL: max_gt_v8i32c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [4294967295,4294967293,4294967293,4294967295] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [7,5,5,7] +; SSE-NEXT: retq +; +; AVX-LABEL: max_gt_v8i32c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [4294967295,4294967293,4294967293,4294967295,7,5,5,7] +; AVX-NEXT: retq + %1 = insertelement <8 x i32> , i32 -7, i32 0 + %2 = insertelement <8 x i32> , i32 -1, i32 0 + %3 = icmp ugt <8 x i32> %1, %2 + %4 = select <8 x i1> %3, <8 x i32> %1, <8 x i32> %2 + ret <8 x i32> %4 +} + +define <8 x i16> @max_gt_v8i16c() { +; SSE-LABEL: max_gt_v8i16c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [65535,65533,65533,65535,7,5,5,7] +; SSE-NEXT: retq +; +; AVX-LABEL: max_gt_v8i16c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [65535,65533,65533,65535,7,5,5,7] +; AVX-NEXT: retq + %1 = insertelement <8 x i16> , i16 -7, i32 0 + %2 = insertelement <8 x i16> , i16 -1, i32 0 + %3 = icmp ugt <8 x i16> %1, %2 + %4 = select <8 x i1> %3, <8 x i16> %1, <8 x i16> %2 + ret <8 x i16> %4 +} + +define <16 x i16> @max_gt_v16i16c() { +; SSE-LABEL: max_gt_v16i16c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [65535,65534,65533,65532,65533,65534,65535,0] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [7,6,5,4,5,6,7,8] +; SSE-NEXT: retq +; +; AVX-LABEL: max_gt_v16i16c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [65535,65534,65533,65532,65533,65534,65535,0,7,6,5,4,5,6,7,8] +; AVX-NEXT: retq + %1 = insertelement <16 x i16> , i16 -7, i32 0 + %2 = insertelement <16 x i16> , i16 -1, i32 0 + %3 = icmp ugt <16 x i16> %1, %2 + %4 = select <16 x i1> %3, <16 x i16> %1, <16 x i16> %2 + ret <16 x i16> %4 +} + +define <16 x i8> @max_gt_v16i8c() { +; SSE-LABEL: max_gt_v16i8c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [255,254,253,252,253,254,255,0,7,6,5,4,5,6,7,8] +; SSE-NEXT: retq +; +; AVX-LABEL: max_gt_v16i8c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [255,254,253,252,253,254,255,0,7,6,5,4,5,6,7,8] +; AVX-NEXT: retq + %1 = insertelement <16 x i8> , i8 -7, i32 0 + %2 = insertelement <16 x i8> , i8 -1, i32 0 + %3 = icmp ugt <16 x i8> %1, %2 + %4 = select <16 x i1> %3, <16 x i8> %1, <16 x i8> %2 + ret <16 x i8> %4 +} + +define <2 x i64> @max_ge_v2i64c() { +; SSE-LABEL: max_ge_v2i64c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [18446744073709551615,7] +; SSE-NEXT: retq +; +; AVX-LABEL: max_ge_v2i64c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [18446744073709551615,7] +; AVX-NEXT: retq + %1 = insertelement <2 x i64> , i64 -7, i32 0 + %2 = insertelement <2 x i64> , i64 -1, i32 0 + %3 = icmp uge <2 x i64> %1, %2 + %4 = select <2 x i1> %3, <2 x i64> %1, <2 x i64> %2 + ret <2 x i64> %4 +} + +define <4 x i64> @max_ge_v4i64c() { +; SSE-LABEL: max_ge_v4i64c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm1 = [7,7] +; SSE-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: max_ge_v4i64c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [18446744073709551615,18446744073709551615,7,7] +; AVX-NEXT: retq + %1 = insertelement <4 x i64> , i64 -7, i32 0 + %2 = insertelement <4 x i64> , i64 -1, i32 0 + %3 = icmp uge <4 x i64> %1, %2 + %4 = select <4 x i1> %3, <4 x i64> %1, <4 x i64> %2 + ret <4 x i64> %4 +} + +define <4 x i32> @max_ge_v4i32c() { +; SSE-LABEL: max_ge_v4i32c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [4294967295,4294967295,7,7] +; SSE-NEXT: retq +; +; AVX-LABEL: max_ge_v4i32c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [4294967295,4294967295,7,7] +; AVX-NEXT: retq + %1 = insertelement <4 x i32> , i32 -7, i32 0 + %2 = insertelement <4 x i32> , i32 -1, i32 0 + %3 = icmp uge <4 x i32> %1, %2 + %4 = select <4 x i1> %3, <4 x i32> %1, <4 x i32> %2 + ret <4 x i32> %4 +} + +define <8 x i32> @max_ge_v8i32c() { +; SSE-LABEL: max_ge_v8i32c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [4294967295,4294967293,4294967293,4294967295] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [7,5,5,7] +; SSE-NEXT: retq +; +; AVX-LABEL: max_ge_v8i32c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [4294967295,4294967293,4294967293,4294967295,7,5,5,7] +; AVX-NEXT: retq + %1 = insertelement <8 x i32> , i32 -7, i32 0 + %2 = insertelement <8 x i32> , i32 -1, i32 0 + %3 = icmp uge <8 x i32> %1, %2 + %4 = select <8 x i1> %3, <8 x i32> %1, <8 x i32> %2 + ret <8 x i32> %4 +} + +define <8 x i16> @max_ge_v8i16c() { +; SSE-LABEL: max_ge_v8i16c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [65535,65533,65533,65535,7,5,5,7] +; SSE-NEXT: retq +; +; AVX-LABEL: max_ge_v8i16c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [65535,65533,65533,65535,7,5,5,7] +; AVX-NEXT: retq + %1 = insertelement <8 x i16> , i16 -7, i32 0 + %2 = insertelement <8 x i16> , i16 -1, i32 0 + %3 = icmp uge <8 x i16> %1, %2 + %4 = select <8 x i1> %3, <8 x i16> %1, <8 x i16> %2 + ret <8 x i16> %4 +} + +define <16 x i16> @max_ge_v16i16c() { +; SSE-LABEL: max_ge_v16i16c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [65535,65534,65533,65532,65533,65534,65535,0] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [7,6,5,4,5,6,7,8] +; SSE-NEXT: retq +; +; AVX-LABEL: max_ge_v16i16c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [65535,65534,65533,65532,65533,65534,65535,0,7,6,5,4,5,6,7,8] +; AVX-NEXT: retq + %1 = insertelement <16 x i16> , i16 -7, i32 0 + %2 = insertelement <16 x i16> , i16 -1, i32 0 + %3 = icmp uge <16 x i16> %1, %2 + %4 = select <16 x i1> %3, <16 x i16> %1, <16 x i16> %2 + ret <16 x i16> %4 +} + +define <16 x i8> @max_ge_v16i8c() { +; SSE-LABEL: max_ge_v16i8c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [255,254,253,252,253,254,255,0,7,6,5,4,5,6,7,8] +; SSE-NEXT: retq +; +; AVX-LABEL: max_ge_v16i8c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [255,254,253,252,253,254,255,0,7,6,5,4,5,6,7,8] +; AVX-NEXT: retq + %1 = insertelement <16 x i8> , i8 -7, i32 0 + %2 = insertelement <16 x i8> , i8 -1, i32 0 + %3 = icmp uge <16 x i8> %1, %2 + %4 = select <16 x i1> %3, <16 x i8> %1, <16 x i8> %2 + ret <16 x i8> %4 +} + +define <2 x i64> @min_lt_v2i64c() { +; SSE-LABEL: min_lt_v2i64c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [18446744073709551609,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_lt_v2i64c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [18446744073709551609,1] +; AVX-NEXT: retq + %1 = insertelement <2 x i64> , i64 -7, i32 0 + %2 = insertelement <2 x i64> , i64 -1, i32 0 + %3 = icmp ult <2 x i64> %1, %2 + %4 = select <2 x i1> %3, <2 x i64> %1, <2 x i64> %2 + ret <2 x i64> %4 +} + +define <4 x i64> @min_lt_v4i64c() { +; SSE-LABEL: min_lt_v4i64c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [18446744073709551609,18446744073709551609] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [1,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_lt_v4i64c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [18446744073709551609,18446744073709551609,1,1] +; AVX-NEXT: retq + %1 = insertelement <4 x i64> , i64 -7, i32 0 + %2 = insertelement <4 x i64> , i64 -1, i32 0 + %3 = icmp ult <4 x i64> %1, %2 + %4 = select <4 x i1> %3, <4 x i64> %1, <4 x i64> %2 + ret <4 x i64> %4 +} + +define <4 x i32> @min_lt_v4i32c() { +; SSE-LABEL: min_lt_v4i32c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [4294967289,4294967289,1,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_lt_v4i32c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [4294967289,4294967289,1,1] +; AVX-NEXT: retq + %1 = insertelement <4 x i32> , i32 -7, i32 0 + %2 = insertelement <4 x i32> , i32 -1, i32 0 + %3 = icmp ult <4 x i32> %1, %2 + %4 = select <4 x i1> %3, <4 x i32> %1, <4 x i32> %2 + ret <4 x i32> %4 +} + +define <8 x i32> @min_lt_v8i32c() { +; SSE-LABEL: min_lt_v8i32c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [4294967289,4294967291,4294967291,4294967289] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [1,3,3,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_lt_v8i32c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [4294967289,4294967291,4294967291,4294967289,1,3,3,1] +; AVX-NEXT: retq + %1 = insertelement <8 x i32> , i32 -7, i32 0 + %2 = insertelement <8 x i32> , i32 -1, i32 0 + %3 = icmp ult <8 x i32> %1, %2 + %4 = select <8 x i1> %3, <8 x i32> %1, <8 x i32> %2 + ret <8 x i32> %4 +} + +define <8 x i16> @min_lt_v8i16c() { +; SSE-LABEL: min_lt_v8i16c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [1,65531,65531,65529,1,3,3,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_lt_v8i16c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [1,65531,65531,65529,1,3,3,1] +; AVX-NEXT: retq + %1 = insertelement <8 x i16> , i16 -7, i32 0 + %2 = insertelement <8 x i16> , i16 1, i32 0 + %3 = icmp ult <8 x i16> %1, %2 + %4 = select <8 x i1> %3, <8 x i16> %1, <8 x i16> %2 + ret <8 x i16> %4 +} + +define <16 x i16> @min_lt_v16i16c() { +; SSE-LABEL: min_lt_v16i16c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [1,65530,65531,65532,65531,65530,65529,0] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [1,2,3,4,3,2,1,0] +; SSE-NEXT: retq +; +; AVX-LABEL: min_lt_v16i16c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [1,65530,65531,65532,65531,65530,65529,0,1,2,3,4,3,2,1,0] +; AVX-NEXT: retq + %1 = insertelement <16 x i16> , i16 -7, i32 0 + %2 = insertelement <16 x i16> , i16 1, i32 0 + %3 = icmp ult <16 x i16> %1, %2 + %4 = select <16 x i1> %3, <16 x i16> %1, <16 x i16> %2 + ret <16 x i16> %4 +} + +define <16 x i8> @min_lt_v16i8c() { +; SSE-LABEL: min_lt_v16i8c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [1,250,251,252,251,250,249,0,1,2,3,4,3,2,1,0] +; SSE-NEXT: retq +; +; AVX-LABEL: min_lt_v16i8c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [1,250,251,252,251,250,249,0,1,2,3,4,3,2,1,0] +; AVX-NEXT: retq + %1 = insertelement <16 x i8> , i8 -7, i32 0 + %2 = insertelement <16 x i8> , i8 1, i32 0 + %3 = icmp ult <16 x i8> %1, %2 + %4 = select <16 x i1> %3, <16 x i8> %1, <16 x i8> %2 + ret <16 x i8> %4 +} + +define <2 x i64> @min_le_v2i64c() { +; SSE-LABEL: min_le_v2i64c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [18446744073709551609,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_le_v2i64c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [18446744073709551609,1] +; AVX-NEXT: retq + %1 = insertelement <2 x i64> , i64 -7, i32 0 + %2 = insertelement <2 x i64> , i64 -1, i32 0 + %3 = icmp ule <2 x i64> %1, %2 + %4 = select <2 x i1> %3, <2 x i64> %1, <2 x i64> %2 + ret <2 x i64> %4 +} + +define <4 x i64> @min_le_v4i64c() { +; SSE-LABEL: min_le_v4i64c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [18446744073709551609,18446744073709551609] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [1,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_le_v4i64c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [18446744073709551609,18446744073709551609,1,1] +; AVX-NEXT: retq + %1 = insertelement <4 x i64> , i64 -7, i32 0 + %2 = insertelement <4 x i64> , i64 -1, i32 0 + %3 = icmp ule <4 x i64> %1, %2 + %4 = select <4 x i1> %3, <4 x i64> %1, <4 x i64> %2 + ret <4 x i64> %4 +} + +define <4 x i32> @min_le_v4i32c() { +; SSE-LABEL: min_le_v4i32c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [4294967289,4294967289,1,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_le_v4i32c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [4294967289,4294967289,1,1] +; AVX-NEXT: retq + %1 = insertelement <4 x i32> , i32 -7, i32 0 + %2 = insertelement <4 x i32> , i32 -1, i32 0 + %3 = icmp ule <4 x i32> %1, %2 + %4 = select <4 x i1> %3, <4 x i32> %1, <4 x i32> %2 + ret <4 x i32> %4 +} + +define <8 x i32> @min_le_v8i32c() { +; SSE-LABEL: min_le_v8i32c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [4294967289,4294967291,4294967291,4294967289] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [1,3,3,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_le_v8i32c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [4294967289,4294967291,4294967291,4294967289,1,3,3,1] +; AVX-NEXT: retq + %1 = insertelement <8 x i32> , i32 -7, i32 0 + %2 = insertelement <8 x i32> , i32 -1, i32 0 + %3 = icmp ule <8 x i32> %1, %2 + %4 = select <8 x i1> %3, <8 x i32> %1, <8 x i32> %2 + ret <8 x i32> %4 +} + +define <8 x i16> @min_le_v8i16c() { +; SSE-LABEL: min_le_v8i16c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [65529,65531,65531,65529,1,3,3,1] +; SSE-NEXT: retq +; +; AVX-LABEL: min_le_v8i16c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [65529,65531,65531,65529,1,3,3,1] +; AVX-NEXT: retq + %1 = insertelement <8 x i16> , i16 -7, i32 0 + %2 = insertelement <8 x i16> , i16 -1, i32 0 + %3 = icmp ule <8 x i16> %1, %2 + %4 = select <8 x i1> %3, <8 x i16> %1, <8 x i16> %2 + ret <8 x i16> %4 +} + +define <16 x i16> @min_le_v16i16c() { +; SSE-LABEL: min_le_v16i16c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [65529,65530,65531,65532,65531,65530,65529,0] +; SSE-NEXT: movaps {{.*#+}} xmm1 = [1,2,3,4,3,2,1,0] +; SSE-NEXT: retq +; +; AVX-LABEL: min_le_v16i16c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [65529,65530,65531,65532,65531,65530,65529,0,1,2,3,4,3,2,1,0] +; AVX-NEXT: retq + %1 = insertelement <16 x i16> , i16 -7, i32 0 + %2 = insertelement <16 x i16> , i16 -1, i32 0 + %3 = icmp ule <16 x i16> %1, %2 + %4 = select <16 x i1> %3, <16 x i16> %1, <16 x i16> %2 + ret <16 x i16> %4 +} + +define <16 x i8> @min_le_v16i8c() { +; SSE-LABEL: min_le_v16i8c: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm0 = [249,250,251,252,251,250,249,0,1,2,3,4,3,2,1,0] +; SSE-NEXT: retq +; +; AVX-LABEL: min_le_v16i8c: +; AVX: # BB#0: +; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [249,250,251,252,251,250,249,0,1,2,3,4,3,2,1,0] +; AVX-NEXT: retq + %1 = insertelement <16 x i8> , i8 -7, i32 0 + %2 = insertelement <16 x i8> , i8 -1, i32 0 + %3 = icmp ule <16 x i8> %1, %2 + %4 = select <16 x i1> %3, <16 x i8> %1, <16 x i8> %2 + ret <16 x i8> %4 +} diff --git a/test/CodeGen/X86/vec_sdiv_to_shift.ll b/test/CodeGen/X86/vec_sdiv_to_shift.ll index 56855d3c44eb..7f71a0c2ea5b 100644 --- a/test/CodeGen/X86/vec_sdiv_to_shift.ll +++ b/test/CodeGen/X86/vec_sdiv_to_shift.ll @@ -13,6 +13,19 @@ entry: ret <8 x i16> %0 } +define <8 x i16> @sdiv_vec8x16_minsize(<8 x i16> %var) minsize { +entry: +; CHECK: sdiv_vec8x16_minsize +; CHECK: psraw $15 +; CHECK: vpsrlw $11 +; CHECK: vpaddw +; CHECK: vpsraw $5 +; CHECK: ret + %0 = sdiv <8 x i16> %var, + ret <8 x i16> %0 +} + + define <4 x i32> @sdiv_zero(<4 x i32> %var) { entry: ; CHECK: sdiv_zero diff --git a/test/CodeGen/X86/vec_trunc_sext.ll b/test/CodeGen/X86/vec_trunc_sext.ll index dcfe423eb748..66af87c78187 100644 --- a/test/CodeGen/X86/vec_trunc_sext.ll +++ b/test/CodeGen/X86/vec_trunc_sext.ll @@ -1,5 +1,5 @@ -; RUN: llc %s -mtriple=x86_64-unknown-unknown -mattr='-sse4.1' -o - | FileCheck %s -check-prefix=NO_SSE_41 -; RUN: llc %s -mtriple=x86_64-unknown-unknown -mattr='+sse4.1' -o - | FileCheck %s -check-prefix=SSE_41 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-sse4.1 | FileCheck %s --check-prefix=NO_SSE_41 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE_41 ; PR20472 ( http://llvm.org/bugs/show_bug.cgi?id=20472 ) ; When sexting a trunc'd vector value, we can't eliminate the zext. @@ -9,22 +9,23 @@ ; but that is beyond our current codegen capabilities. define <4 x i32> @trunc_sext(<4 x i16>* %in) { +; NO_SSE_41-LABEL: trunc_sext: +; NO_SSE_41: # BB#0: +; NO_SSE_41-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; NO_SSE_41-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; NO_SSE_41-NEXT: pslld $24, %xmm0 +; NO_SSE_41-NEXT: psrad $24, %xmm0 +; NO_SSE_41-NEXT: retq +; +; SSE_41-LABEL: trunc_sext: +; SSE_41: # BB#0: +; SSE_41-NEXT: pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero +; SSE_41-NEXT: pslld $24, %xmm0 +; SSE_41-NEXT: psrad $24, %xmm0 +; SSE_41-NEXT: retq %load = load <4 x i16>, <4 x i16>* %in %trunc = trunc <4 x i16> %load to <4 x i8> %sext = sext <4 x i8> %trunc to <4 x i32> ret <4 x i32> %sext - -; NO_SSE_41-LABEL: trunc_sext: -; NO_SSE_41: movq (%rdi), %xmm0 -; NO_SSE_41-NEXT: punpcklwd %xmm0, %xmm0 -; NO_SSE_41-NEXT: pslld $24, %xmm0 -; NO_SSE_41-NEXT: psrad $24, %xmm0 -; NO_SSE_41-NEXT: retq - -; SSE_41-LABEL: trunc_sext: -; SSE_41: pmovzxwd (%rdi), %xmm0 -; SSE_41-NEXT: pslld $24, %xmm0 -; SSE_41-NEXT: psrad $24, %xmm0 -; SSE_41-NEXT: retq } diff --git a/test/CodeGen/X86/vec_uint_to_fp-fastmath.ll b/test/CodeGen/X86/vec_uint_to_fp-fastmath.ll new file mode 100644 index 000000000000..1f36d064f873 --- /dev/null +++ b/test/CodeGen/X86/vec_uint_to_fp-fastmath.ll @@ -0,0 +1,130 @@ +; RUN: llc < %s -mtriple=x86_64 -enable-unsafe-fp-math \ +; RUN: | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=CST +; RUN: llc < %s -mtriple=x86_64 -enable-unsafe-fp-math -mattr=+sse4.1 \ +; RUN: | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=CST +; RUN: llc < %s -mtriple=x86_64 -enable-unsafe-fp-math -mattr=+avx \ +; RUN: | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=CST +; RUN: llc < %s -mtriple=x86_64 -enable-unsafe-fp-math -mattr=+avx2 \ +; RUN: | FileCheck %s --check-prefix=CHECK --check-prefix=AVX2 + +; CST: [[MASKCSTADDR:.LCPI[0-9_]+]]: +; CST-NEXT: .long 65535 # 0xffff +; CST-NEXT: .long 65535 # 0xffff +; CST-NEXT: .long 65535 # 0xffff +; CST-NEXT: .long 65535 # 0xffff + +; CST: [[FPMASKCSTADDR:.LCPI[0-9_]+]]: +; CST-NEXT: .long 1199570944 # float 65536 +; CST-NEXT: .long 1199570944 # float 65536 +; CST-NEXT: .long 1199570944 # float 65536 +; CST-NEXT: .long 1199570944 # float 65536 + +; AVX2: [[FPMASKCSTADDR:.LCPI[0-9_]+]]: +; AVX2-NEXT: .long 1199570944 # float 65536 + +; AVX2: [[MASKCSTADDR:.LCPI[0-9_]+]]: +; AVX2-NEXT: .long 65535 # 0xffff + +define <4 x float> @test_uitofp_v4i32_to_v4f32(<4 x i32> %arg) { +; SSE-LABEL: test_uitofp_v4i32_to_v4f32: +; SSE: # BB#0: +; SSE-NEXT: movaps {{.*#+}} xmm1 = [65535,65535,65535,65535] +; SSE-NEXT: andps %xmm0, %xmm1 +; SSE-NEXT: cvtdq2ps %xmm1, %xmm1 +; SSE-NEXT: psrld $16, %xmm0 +; SSE-NEXT: cvtdq2ps %xmm0, %xmm0 +; SSE-NEXT: mulps [[FPMASKCSTADDR]](%rip), %xmm0 +; SSE-NEXT: addps %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test_uitofp_v4i32_to_v4f32: +; AVX: # BB#0: +; AVX-NEXT: vandps [[MASKCSTADDR]](%rip), %xmm0, %xmm1 +; AVX-NEXT: vcvtdq2ps %xmm1, %xmm1 +; AVX-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX-NEXT: vcvtdq2ps %xmm0, %xmm0 +; AVX-NEXT: vmulps [[FPMASKCSTADDR]](%rip), %xmm0, %xmm0 +; AVX-NEXT: vaddps %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; AVX2-LABEL: test_uitofp_v4i32_to_v4f32: +; AVX2: # BB#0: +; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX2-NEXT: vcvtdq2ps %xmm1, %xmm1 +; AVX2-NEXT: vbroadcastss [[FPMASKCSTADDR]](%rip), %xmm2 +; AVX2-NEXT: vmulps %xmm2, %xmm1, %xmm1 +; AVX2-NEXT: vpbroadcastd [[MASKCSTADDR]](%rip), %xmm2 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vcvtdq2ps %xmm0, %xmm0 +; AVX2-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: retq + %tmp = uitofp <4 x i32> %arg to <4 x float> + ret <4 x float> %tmp +} + +; AVX: [[MASKCSTADDR_v8:.LCPI[0-9_]+]]: +; AVX-NEXT: .long 65535 # 0xffff +; AVX-NEXT: .long 65535 # 0xffff +; AVX-NEXT: .long 65535 # 0xffff +; AVX-NEXT: .long 65535 # 0xffff + +; AVX: [[FPMASKCSTADDR_v8:.LCPI[0-9_]+]]: +; AVX-NEXT: .long 1199570944 # float 65536 +; AVX-NEXT: .long 1199570944 # float 65536 +; AVX-NEXT: .long 1199570944 # float 65536 +; AVX-NEXT: .long 1199570944 # float 65536 + +; AVX2: [[FPMASKCSTADDR_v8:.LCPI[0-9_]+]]: +; AVX2-NEXT: .long 1199570944 # float 65536 + +; AVX2: [[MASKCSTADDR_v8:.LCPI[0-9_]+]]: +; AVX2-NEXT: .long 65535 # 0xffff + +define <8 x float> @test_uitofp_v8i32_to_v8f32(<8 x i32> %arg) { +; SSE-LABEL: test_uitofp_v8i32_to_v8f32: +; SSE: # BB#0: +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: psrld $16, %xmm2 +; SSE-NEXT: cvtdq2ps %xmm2, %xmm2 +; SSE-NEXT: movaps {{.*#+}} xmm3 = [6.553600e+04,6.553600e+04,6.553600e+04,6.553600e+04] +; SSE-NEXT: mulps %xmm3, %xmm2 +; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,65535,65535] +; SSE-NEXT: pand %xmm4, %xmm0 +; SSE-NEXT: cvtdq2ps %xmm0, %xmm0 +; SSE-NEXT: addps %xmm2, %xmm0 +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: psrld $16, %xmm2 +; SSE-NEXT: cvtdq2ps %xmm2, %xmm2 +; SSE-NEXT: mulps %xmm3, %xmm2 +; SSE-NEXT: pand %xmm4, %xmm1 +; SSE-NEXT: cvtdq2ps %xmm1, %xmm1 +; SSE-NEXT: addps %xmm2, %xmm1 +; SSE-NEXT: retq +; +; AVX-LABEL: test_uitofp_v8i32_to_v8f32: +; AVX: # BB#0: +; AVX-NEXT: vandps [[MASKCSTADDR_v8]](%rip), %ymm0, %ymm1 +; AVX-NEXT: vcvtdq2ps %ymm1, %ymm1 +; AVX-NEXT: vpsrld $16, %xmm0, %xmm2 +; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0 +; AVX-NEXT: vcvtdq2ps %ymm0, %ymm0 +; AVX-NEXT: vmulps [[FPMASKCSTADDR_v8]](%rip), %ymm0, %ymm0 +; AVX-NEXT: vaddps %ymm1, %ymm0, %ymm0 +; AVX-NEXT: retq +; +; AVX2-LABEL: test_uitofp_v8i32_to_v8f32: +; AVX2: # BB#0: +; AVX2-NEXT: vpsrld $16, %ymm0, %ymm1 +; AVX2-NEXT: vcvtdq2ps %ymm1, %ymm1 +; AVX2-NEXT: vbroadcastss [[FPMASKCSTADDR_v8]](%rip), %ymm2 +; AVX2-NEXT: vmulps %ymm2, %ymm1, %ymm1 +; AVX2-NEXT: vpbroadcastd [[MASKCSTADDR_v8]](%rip), %ymm2 +; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vcvtdq2ps %ymm0, %ymm0 +; AVX2-NEXT: vaddps %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq + %tmp = uitofp <8 x i32> %arg to <8 x float> + ret <8 x float> %tmp +} diff --git a/test/CodeGen/X86/vec_uint_to_fp.ll b/test/CodeGen/X86/vec_uint_to_fp.ll index 46cfcd9a9a12..ce0c11b2fa2a 100644 --- a/test/CodeGen/X86/vec_uint_to_fp.ll +++ b/test/CodeGen/X86/vec_uint_to_fp.ll @@ -23,10 +23,10 @@ ; CST-NEXT: .long 1392508928 ## 0x53000000 ; CST: [[MAGICCSTADDR:LCPI0_[0-9]+]]: -; CST-NEXT: .long 3539992704 ## float -5.497642e+11 -; CST-NEXT: .long 3539992704 ## float -5.497642e+11 -; CST-NEXT: .long 3539992704 ## float -5.497642e+11 -; CST-NEXT: .long 3539992704 ## float -5.497642e+11 +; CST-NEXT: .long 3539992704 ## float -5.49764202E+11 +; CST-NEXT: .long 3539992704 ## float -5.49764202E+11 +; CST-NEXT: .long 3539992704 ## float -5.49764202E+11 +; CST-NEXT: .long 3539992704 ## float -5.49764202E+11 ; AVX2: [[LOWCSTADDR:LCPI0_[0-9]+]]: ; AVX2-NEXT: .long 1258291200 ## 0x4b000000 diff --git a/test/CodeGen/X86/vector-blend.ll b/test/CodeGen/X86/vector-blend.ll index e15daaa54a33..aaf81f2f9bb6 100644 --- a/test/CodeGen/X86/vector-blend.ll +++ b/test/CodeGen/X86/vector-blend.ll @@ -255,31 +255,32 @@ entry: define <16 x i8> @vsel_i8(<16 x i8> %v1, <16 x i8> %v2) { ; SSE2-LABEL: vsel_i8: ; SSE2: # BB#0: # %entry -; SSE2-NEXT: movaps {{.*#+}} xmm2 = [255,0,0,0,255,0,0,0,255,255,255,255,255,255,255,255] -; SSE2-NEXT: andps %xmm2, %xmm0 -; SSE2-NEXT: andnps %xmm1, %xmm2 -; SSE2-NEXT: orps %xmm2, %xmm0 +; SSE2-NEXT: movaps {{.*#+}} xmm2 = [0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255] +; SSE2-NEXT: andps %xmm2, %xmm1 +; SSE2-NEXT: andnps %xmm0, %xmm2 +; SSE2-NEXT: orps %xmm1, %xmm2 +; SSE2-NEXT: movaps %xmm2, %xmm0 ; SSE2-NEXT: retq ; ; SSSE3-LABEL: vsel_i8: ; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: pshufb {{.*#+}} xmm1 = zero,xmm1[1,2,3],zero,xmm1[5,6,7],zero,zero,zero,zero,zero,zero,zero,zero -; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[8,9,10,11,12,13,14,15] +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[12],zero,zero,zero +; SSSE3-NEXT: pshufb {{.*#+}} xmm1 = zero,xmm1[1,2,3],zero,xmm1[5,6,7],zero,xmm1[9,10,11],zero,xmm1[13,14,15] ; SSSE3-NEXT: por %xmm1, %xmm0 ; SSSE3-NEXT: retq ; ; SSE41-LABEL: vsel_i8: ; SSE41: # BB#0: # %entry ; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: movaps {{.*#+}} xmm0 = [255,0,0,0,255,0,0,0,255,255,255,255,255,255,255,255] -; SSE41-NEXT: pblendvb %xmm2, %xmm1 -; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: movaps {{.*#+}} xmm0 = [0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255] +; SSE41-NEXT: pblendvb %xmm1, %xmm2 +; SSE41-NEXT: movdqa %xmm2, %xmm0 ; SSE41-NEXT: retq ; ; AVX-LABEL: vsel_i8: ; AVX: # BB#0: # %entry -; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [255,0,0,0,255,0,0,0,255,255,255,255,255,255,255,255] -; AVX-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0 +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255] +; AVX-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq entry: %vsel = select <16 x i1> , <16 x i8> %v1, <16 x i8> %v2 @@ -623,49 +624,52 @@ entry: define <32 x i8> @constant_pblendvb_avx2(<32 x i8> %xyzw, <32 x i8> %abcd) { ; SSE2-LABEL: constant_pblendvb_avx2: ; SSE2: # BB#0: # %entry -; SSE2-NEXT: movaps {{.*#+}} xmm4 = [0,0,255,0,255,255,255,0,255,255,255,255,255,255,255,255] +; SSE2-NEXT: movaps {{.*#+}} xmm4 = [255,255,0,255,0,0,0,255,255,255,0,255,0,0,0,255] ; SSE2-NEXT: movaps %xmm4, %xmm5 -; SSE2-NEXT: andnps %xmm2, %xmm5 -; SSE2-NEXT: andps %xmm4, %xmm0 -; SSE2-NEXT: orps %xmm5, %xmm0 -; SSE2-NEXT: andps %xmm4, %xmm1 -; SSE2-NEXT: andnps %xmm3, %xmm4 -; SSE2-NEXT: orps %xmm4, %xmm1 +; SSE2-NEXT: andnps %xmm0, %xmm5 +; SSE2-NEXT: andps %xmm4, %xmm2 +; SSE2-NEXT: orps %xmm2, %xmm5 +; SSE2-NEXT: andps %xmm4, %xmm3 +; SSE2-NEXT: andnps %xmm1, %xmm4 +; SSE2-NEXT: orps %xmm3, %xmm4 +; SSE2-NEXT: movaps %xmm5, %xmm0 +; SSE2-NEXT: movaps %xmm4, %xmm1 ; SSE2-NEXT: retq ; ; SSSE3-LABEL: constant_pblendvb_avx2: ; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: movdqa {{.*#+}} xmm4 = [0,1,128,3,128,128,128,7,128,128,128,128,128,128,128,128] -; SSSE3-NEXT: pshufb %xmm4, %xmm2 -; SSSE3-NEXT: movdqa {{.*#+}} xmm5 = [128,128,2,128,4,5,6,128,8,9,10,11,12,13,14,15] -; SSSE3-NEXT: pshufb %xmm5, %xmm0 +; SSSE3-NEXT: movdqa {{.*#+}} xmm4 = [128,128,2,128,4,5,6,128,128,128,10,128,12,13,14,128] +; SSSE3-NEXT: pshufb %xmm4, %xmm0 +; SSSE3-NEXT: movdqa {{.*#+}} xmm5 = [0,1,128,3,128,128,128,7,8,9,128,11,128,128,128,15] +; SSSE3-NEXT: pshufb %xmm5, %xmm2 ; SSSE3-NEXT: por %xmm2, %xmm0 -; SSSE3-NEXT: pshufb %xmm4, %xmm3 -; SSSE3-NEXT: pshufb %xmm5, %xmm1 +; SSSE3-NEXT: pshufb %xmm4, %xmm1 +; SSSE3-NEXT: pshufb %xmm5, %xmm3 ; SSSE3-NEXT: por %xmm3, %xmm1 ; SSSE3-NEXT: retq ; ; SSE41-LABEL: constant_pblendvb_avx2: ; SSE41: # BB#0: # %entry ; SSE41-NEXT: movdqa %xmm0, %xmm4 -; SSE41-NEXT: movaps {{.*#+}} xmm0 = [0,0,255,0,255,255,255,0,255,255,255,255,255,255,255,255] -; SSE41-NEXT: pblendvb %xmm4, %xmm2 -; SSE41-NEXT: pblendvb %xmm1, %xmm3 -; SSE41-NEXT: movdqa %xmm2, %xmm0 -; SSE41-NEXT: movdqa %xmm3, %xmm1 +; SSE41-NEXT: movaps {{.*#+}} xmm0 = [255,255,0,255,0,0,0,255,255,255,0,255,0,0,0,255] +; SSE41-NEXT: pblendvb %xmm2, %xmm4 +; SSE41-NEXT: pblendvb %xmm3, %xmm1 +; SSE41-NEXT: movdqa %xmm4, %xmm0 ; SSE41-NEXT: retq ; ; AVX1-LABEL: constant_pblendvb_avx2: ; AVX1: # BB#0: # %entry -; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [0,0,255,0,255,255,255,0,255,255,255,255,255,255,255,255] -; AVX1-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm1 -; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 -; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vmovdqa .LCPI18_0(%rip), %xmm4 # xmm4 = [255,255,0,255,0,0,0,255,255,255,0,255,0,0,0,255] +; AVX1-NEXT: vpblendvb %xmm4, %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpblendvb %xmm4, %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: constant_pblendvb_avx2: ; AVX2: # BB#0: # %entry -; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [0,0,255,0,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] +; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [0,0,255,0,255,255,255,0,0,0,255,0,255,255,255,0,0,0,255,0,255,255,255,0,0,0,255,0,255,255,255,0] ; AVX2-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0 ; AVX2-NEXT: retq entry: diff --git a/test/CodeGen/X86/vector-idiv.ll b/test/CodeGen/X86/vector-idiv.ll index 2e482a0f1430..1117e206e5b0 100644 --- a/test/CodeGen/X86/vector-idiv.ll +++ b/test/CodeGen/X86/vector-idiv.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -march=x86-64 -mcpu=core2 -mattr=+sse4.1 < %s | FileCheck %s --check-prefix=SSE41 ; RUN: llc -march=x86-64 -mcpu=core2 < %s | FileCheck %s --check-prefix=SSE ; RUN: llc -march=x86-64 -mcpu=core-avx2 < %s | FileCheck %s --check-prefix=AVX diff --git a/test/CodeGen/X86/vector-lzcnt-128.ll b/test/CodeGen/X86/vector-lzcnt-128.ll index b43188b7c6ea..8bf0af68e6dc 100644 --- a/test/CodeGen/X86/vector-lzcnt-128.ll +++ b/test/CodeGen/X86/vector-lzcnt-128.ll @@ -1,13 +1,13 @@ -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE3 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE3 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512cd -mattr=+avx512vl | FileCheck %s --check-prefix=AVX512VLCD --check-prefix=ALL --check-prefix=AVX512 +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512cd | FileCheck %s --check-prefix=AVX512CD --check-prefix=ALL --check-prefix=AVX512 -target triple = "x86_64-unknown-unknown" - -define <2 x i64> @testv2i64(<2 x i64> %in) { +define <2 x i64> @testv2i64(<2 x i64> %in) nounwind { ; SSE2-LABEL: testv2i64: ; SSE2: # BB#0: ; SSE2-NEXT: movd %xmm0, %rax @@ -16,13 +16,13 @@ define <2 x i64> @testv2i64(<2 x i64> %in) { ; SSE2-NEXT: cmoveq %rcx, %rax ; SSE2-NEXT: xorq $63, %rax ; SSE2-NEXT: movd %rax, %xmm1 -; SSE2-NEXT: pshufd $78, %xmm0, %xmm0 # xmm0 = xmm0[2,3,0,1] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] ; SSE2-NEXT: movd %xmm0, %rax ; SSE2-NEXT: bsrq %rax, %rax ; SSE2-NEXT: cmoveq %rcx, %rax ; SSE2-NEXT: xorq $63, %rax ; SSE2-NEXT: movd %rax, %xmm0 -; SSE2-NEXT: punpcklqdq %xmm0, %xmm1 # xmm1 = xmm1[0],xmm0[0] +; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] ; SSE2-NEXT: movdqa %xmm1, %xmm0 ; SSE2-NEXT: retq ; @@ -34,13 +34,13 @@ define <2 x i64> @testv2i64(<2 x i64> %in) { ; SSE3-NEXT: cmoveq %rcx, %rax ; SSE3-NEXT: xorq $63, %rax ; SSE3-NEXT: movd %rax, %xmm1 -; SSE3-NEXT: pshufd $78, %xmm0, %xmm0 # xmm0 = xmm0[2,3,0,1] +; SSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] ; SSE3-NEXT: movd %xmm0, %rax ; SSE3-NEXT: bsrq %rax, %rax ; SSE3-NEXT: cmoveq %rcx, %rax ; SSE3-NEXT: xorq $63, %rax ; SSE3-NEXT: movd %rax, %xmm0 -; SSE3-NEXT: punpcklqdq %xmm0, %xmm1 # xmm1 = xmm1[0],xmm0[0] +; SSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] ; SSE3-NEXT: movdqa %xmm1, %xmm0 ; SSE3-NEXT: retq ; @@ -52,16 +52,15 @@ define <2 x i64> @testv2i64(<2 x i64> %in) { ; SSSE3-NEXT: cmoveq %rcx, %rax ; SSSE3-NEXT: xorq $63, %rax ; SSSE3-NEXT: movd %rax, %xmm1 -; SSSE3-NEXT: pshufd $78, %xmm0, %xmm0 # xmm0 = xmm0[2,3,0,1] +; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] ; SSSE3-NEXT: movd %xmm0, %rax ; SSSE3-NEXT: bsrq %rax, %rax ; SSSE3-NEXT: cmoveq %rcx, %rax ; SSSE3-NEXT: xorq $63, %rax ; SSSE3-NEXT: movd %rax, %xmm0 -; SSSE3-NEXT: punpcklqdq %xmm0, %xmm1 # xmm1 = xmm1[0],xmm0[0] +; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] ; SSSE3-NEXT: movdqa %xmm1, %xmm0 ; SSSE3-NEXT: retq - ; ; SSE41-LABEL: testv2i64: ; SSE41: # BB#0: @@ -94,11 +93,22 @@ define <2 x i64> @testv2i64(<2 x i64> %in) { ; AVX-NEXT: vmovq %rax, %xmm0 ; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: testv2i64: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vplzcntq %xmm0, %xmm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: testv2i64: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vplzcntq %zmm0, %zmm0 +; AVX512CD-NEXT: retq + %out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %in, i1 0) ret <2 x i64> %out } -define <2 x i64> @testv2i64u(<2 x i64> %in) { +define <2 x i64> @testv2i64u(<2 x i64> %in) nounwind { ; SSE2-LABEL: testv2i64u: ; SSE2: # BB#0: ; SSE2-NEXT: movd %xmm0, %rax @@ -169,11 +179,22 @@ define <2 x i64> @testv2i64u(<2 x i64> %in) { ; AVX-NEXT: vmovq %rax, %xmm0 ; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: testv2i64u: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vplzcntq %xmm0, %xmm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: testv2i64u: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vplzcntq %zmm0, %zmm0 +; AVX512CD-NEXT: retq + %out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %in, i1 -1) ret <2 x i64> %out } -define <4 x i32> @testv4i32(<4 x i32> %in) { +define <4 x i32> @testv4i32(<4 x i32> %in) nounwind { ; SSE2-LABEL: testv4i32: ; SSE2: # BB#0: ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3] @@ -320,11 +341,22 @@ define <4 x i32> @testv4i32(<4 x i32> %in) { ; AVX-NEXT: xorl $31, %eax ; AVX-NEXT: vpinsrd $3, %eax, %xmm1, %xmm0 ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: testv4i32: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vplzcntd %xmm0, %xmm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: testv4i32: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512CD-NEXT: retq + %out = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %in, i1 0) ret <4 x i32> %out } -define <4 x i32> @testv4i32u(<4 x i32> %in) { +define <4 x i32> @testv4i32u(<4 x i32> %in) nounwind { ; SSE2-LABEL: testv4i32u: ; SSE2: # BB#0: ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3] @@ -446,11 +478,22 @@ define <4 x i32> @testv4i32u(<4 x i32> %in) { ; AVX-NEXT: xorl $31, %eax ; AVX-NEXT: vpinsrd $3, %eax, %xmm1, %xmm0 ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: testv4i32u: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vplzcntd %xmm0, %xmm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: testv4i32u: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512CD-NEXT: retq + %out = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %in, i1 -1) ret <4 x i32> %out } -define <8 x i16> @testv8i16(<8 x i16> %in) { +define <8 x i16> @testv8i16(<8 x i16> %in) nounwind { ; SSE2-LABEL: testv8i16: ; SSE2: # BB#0: ; SSE2-NEXT: pextrw $7, %xmm0, %eax @@ -697,11 +740,27 @@ define <8 x i16> @testv8i16(<8 x i16> %in) { ; AVX-NEXT: xorl $15, %ecx ; AVX-NEXT: vpinsrw $7, %ecx, %xmm1, %xmm0 ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: testv8i16: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vpmovzxwd %xmm0, %ymm0 +; AVX512VLCD-NEXT: vplzcntd %ymm0, %ymm0 +; AVX512VLCD-NEXT: vpmovdw %ymm0, %xmm0 +; AVX512VLCD-NEXT: vpsubw {{.*}}(%rip), %xmm0, %xmm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: testv8i16: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512CD-NEXT: vpmovdw %zmm0, %ymm0 +; AVX512CD-NEXT: vpsubw {{.*}}(%rip), %xmm0, %xmm0 +; AVX512CD-NEXT: retq %out = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %in, i1 0) ret <8 x i16> %out } -define <8 x i16> @testv8i16u(<8 x i16> %in) { +define <8 x i16> @testv8i16u(<8 x i16> %in) nounwind { ; SSE2-LABEL: testv8i16u: ; SSE2: # BB#0: ; SSE2-NEXT: pextrw $7, %xmm0, %eax @@ -903,29 +962,46 @@ define <8 x i16> @testv8i16u(<8 x i16> %in) { ; AVX-NEXT: xorl $15, %eax ; AVX-NEXT: vpinsrw $7, %eax, %xmm1, %xmm0 ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: testv8i16u: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vpmovzxwd %xmm0, %ymm0 +; AVX512VLCD-NEXT: vplzcntd %ymm0, %ymm0 +; AVX512VLCD-NEXT: vpmovdw %ymm0, %xmm0 +; AVX512VLCD-NEXT: vpsubw {{.*}}(%rip), %xmm0, %xmm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: testv8i16u: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512CD-NEXT: vpmovdw %zmm0, %ymm0 +; AVX512CD-NEXT: vpsubw {{.*}}(%rip), %xmm0, %xmm0 +; AVX512CD-NEXT: retq %out = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %in, i1 -1) ret <8 x i16> %out } -define <16 x i8> @testv16i8(<16 x i8> %in) { +define <16 x i8> @testv16i8(<16 x i8> %in) nounwind { ; SSE2-LABEL: testv16i8: ; SSE2: # BB#0: -; SSE2: pushq %rbp -; SSE2: movaps %xmm0, -24(%rsp) -; SSE2-NEXT: movzbl -9(%rsp), %eax +; SSE2-NEXT: pushq %rbp +; SSE2-NEXT: pushq %rbx +; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp) +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax ; SSE2-NEXT: bsrl %eax, %ecx ; SSE2-NEXT: movl $15, %eax ; SSE2-NEXT: cmovel %eax, %ecx ; SSE2-NEXT: xorl $7, %ecx ; SSE2-NEXT: movd %ecx, %xmm0 -; SSE2-NEXT: movzbl -10(%rsp), %ebx -; SSE2-NEXT: movzbl -11(%rsp), %edi -; SSE2-NEXT: movzbl -12(%rsp), %r9d -; SSE2-NEXT: movzbl -13(%rsp), %edx -; SSE2-NEXT: movzbl -14(%rsp), %r11d -; SSE2-NEXT: movzbl -15(%rsp), %esi -; SSE2-NEXT: movzbl -16(%rsp), %r8d -; SSE2-NEXT: movzbl -17(%rsp), %ecx +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebx +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edi +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r9d +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r11d +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r8d +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx ; SSE2-NEXT: bsrl %ecx, %ecx ; SSE2-NEXT: cmovel %eax, %ecx ; SSE2-NEXT: xorl $7, %ecx @@ -935,10 +1011,10 @@ define <16 x i8> @testv16i8(<16 x i8> %in) { ; SSE2-NEXT: cmovel %eax, %ecx ; SSE2-NEXT: xorl $7, %ecx ; SSE2-NEXT: movd %ecx, %xmm2 -; SSE2-NEXT: movzbl -18(%rsp), %edx -; SSE2-NEXT: movzbl -19(%rsp), %ecx -; SSE2-NEXT: movzbl -20(%rsp), %r10d -; SSE2-NEXT: movzbl -21(%rsp), %ebp +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r10d +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebp ; SSE2-NEXT: bsrl %ebp, %ebp ; SSE2-NEXT: cmovel %eax, %ebp ; SSE2-NEXT: xorl $7, %ebp @@ -958,8 +1034,8 @@ define <16 x i8> @testv16i8(<16 x i8> %in) { ; SSE2-NEXT: cmovel %eax, %ecx ; SSE2-NEXT: xorl $7, %ecx ; SSE2-NEXT: movd %ecx, %xmm3 -; SSE2-NEXT: movzbl -22(%rsp), %esi -; SSE2-NEXT: movzbl -23(%rsp), %ecx +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx ; SSE2-NEXT: bsrl %ecx, %ecx ; SSE2-NEXT: cmovel %eax, %ecx ; SSE2-NEXT: xorl $7, %ecx @@ -999,7 +1075,7 @@ define <16 x i8> @testv16i8(<16 x i8> %in) { ; SSE2-NEXT: cmovel %eax, %ecx ; SSE2-NEXT: xorl $7, %ecx ; SSE2-NEXT: movd %ecx, %xmm4 -; SSE2-NEXT: movzbl -24(%rsp), %ecx +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx ; SSE2-NEXT: bsrl %ecx, %ecx ; SSE2-NEXT: cmovel %eax, %ecx ; SSE2-NEXT: xorl $7, %ecx @@ -1014,22 +1090,23 @@ define <16 x i8> @testv16i8(<16 x i8> %in) { ; ; SSE3-LABEL: testv16i8: ; SSE3: # BB#0: -; SSE3: pushq %rbp -; SSE3: movaps %xmm0, -24(%rsp) -; SSE3-NEXT: movzbl -9(%rsp), %eax +; SSE3-NEXT: pushq %rbp +; SSE3-NEXT: pushq %rbx +; SSE3-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp) +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax ; SSE3-NEXT: bsrl %eax, %ecx ; SSE3-NEXT: movl $15, %eax ; SSE3-NEXT: cmovel %eax, %ecx ; SSE3-NEXT: xorl $7, %ecx ; SSE3-NEXT: movd %ecx, %xmm0 -; SSE3-NEXT: movzbl -10(%rsp), %ebx -; SSE3-NEXT: movzbl -11(%rsp), %edi -; SSE3-NEXT: movzbl -12(%rsp), %r9d -; SSE3-NEXT: movzbl -13(%rsp), %edx -; SSE3-NEXT: movzbl -14(%rsp), %r11d -; SSE3-NEXT: movzbl -15(%rsp), %esi -; SSE3-NEXT: movzbl -16(%rsp), %r8d -; SSE3-NEXT: movzbl -17(%rsp), %ecx +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebx +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %edi +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %r9d +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %r11d +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %r8d +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx ; SSE3-NEXT: bsrl %ecx, %ecx ; SSE3-NEXT: cmovel %eax, %ecx ; SSE3-NEXT: xorl $7, %ecx @@ -1039,10 +1116,10 @@ define <16 x i8> @testv16i8(<16 x i8> %in) { ; SSE3-NEXT: cmovel %eax, %ecx ; SSE3-NEXT: xorl $7, %ecx ; SSE3-NEXT: movd %ecx, %xmm2 -; SSE3-NEXT: movzbl -18(%rsp), %edx -; SSE3-NEXT: movzbl -19(%rsp), %ecx -; SSE3-NEXT: movzbl -20(%rsp), %r10d -; SSE3-NEXT: movzbl -21(%rsp), %ebp +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %r10d +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebp ; SSE3-NEXT: bsrl %ebp, %ebp ; SSE3-NEXT: cmovel %eax, %ebp ; SSE3-NEXT: xorl $7, %ebp @@ -1062,8 +1139,8 @@ define <16 x i8> @testv16i8(<16 x i8> %in) { ; SSE3-NEXT: cmovel %eax, %ecx ; SSE3-NEXT: xorl $7, %ecx ; SSE3-NEXT: movd %ecx, %xmm3 -; SSE3-NEXT: movzbl -22(%rsp), %esi -; SSE3-NEXT: movzbl -23(%rsp), %ecx +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx ; SSE3-NEXT: bsrl %ecx, %ecx ; SSE3-NEXT: cmovel %eax, %ecx ; SSE3-NEXT: xorl $7, %ecx @@ -1103,7 +1180,7 @@ define <16 x i8> @testv16i8(<16 x i8> %in) { ; SSE3-NEXT: cmovel %eax, %ecx ; SSE3-NEXT: xorl $7, %ecx ; SSE3-NEXT: movd %ecx, %xmm4 -; SSE3-NEXT: movzbl -24(%rsp), %ecx +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx ; SSE3-NEXT: bsrl %ecx, %ecx ; SSE3-NEXT: cmovel %eax, %ecx ; SSE3-NEXT: xorl $7, %ecx @@ -1118,22 +1195,23 @@ define <16 x i8> @testv16i8(<16 x i8> %in) { ; ; SSSE3-LABEL: testv16i8: ; SSSE3: # BB#0: -; SSSE3: pushq %rbp -; SSSE3: movaps %xmm0, -24(%rsp) -; SSSE3-NEXT: movzbl -9(%rsp), %eax +; SSSE3-NEXT: pushq %rbp +; SSSE3-NEXT: pushq %rbx +; SSSE3-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp) +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax ; SSSE3-NEXT: bsrl %eax, %ecx ; SSSE3-NEXT: movl $15, %eax ; SSSE3-NEXT: cmovel %eax, %ecx ; SSSE3-NEXT: xorl $7, %ecx ; SSSE3-NEXT: movd %ecx, %xmm0 -; SSSE3-NEXT: movzbl -10(%rsp), %ebx -; SSSE3-NEXT: movzbl -11(%rsp), %edi -; SSSE3-NEXT: movzbl -12(%rsp), %r9d -; SSSE3-NEXT: movzbl -13(%rsp), %edx -; SSSE3-NEXT: movzbl -14(%rsp), %r11d -; SSSE3-NEXT: movzbl -15(%rsp), %esi -; SSSE3-NEXT: movzbl -16(%rsp), %r8d -; SSSE3-NEXT: movzbl -17(%rsp), %ecx +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebx +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %edi +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %r9d +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %r11d +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %r8d +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx ; SSSE3-NEXT: bsrl %ecx, %ecx ; SSSE3-NEXT: cmovel %eax, %ecx ; SSSE3-NEXT: xorl $7, %ecx @@ -1143,10 +1221,10 @@ define <16 x i8> @testv16i8(<16 x i8> %in) { ; SSSE3-NEXT: cmovel %eax, %ecx ; SSSE3-NEXT: xorl $7, %ecx ; SSSE3-NEXT: movd %ecx, %xmm2 -; SSSE3-NEXT: movzbl -18(%rsp), %edx -; SSSE3-NEXT: movzbl -19(%rsp), %ecx -; SSSE3-NEXT: movzbl -20(%rsp), %r10d -; SSSE3-NEXT: movzbl -21(%rsp), %ebp +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %r10d +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebp ; SSSE3-NEXT: bsrl %ebp, %ebp ; SSSE3-NEXT: cmovel %eax, %ebp ; SSSE3-NEXT: xorl $7, %ebp @@ -1166,8 +1244,8 @@ define <16 x i8> @testv16i8(<16 x i8> %in) { ; SSSE3-NEXT: cmovel %eax, %ecx ; SSSE3-NEXT: xorl $7, %ecx ; SSSE3-NEXT: movd %ecx, %xmm3 -; SSSE3-NEXT: movzbl -22(%rsp), %esi -; SSSE3-NEXT: movzbl -23(%rsp), %ecx +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx ; SSSE3-NEXT: bsrl %ecx, %ecx ; SSSE3-NEXT: cmovel %eax, %ecx ; SSSE3-NEXT: xorl $7, %ecx @@ -1207,7 +1285,7 @@ define <16 x i8> @testv16i8(<16 x i8> %in) { ; SSSE3-NEXT: cmovel %eax, %ecx ; SSSE3-NEXT: xorl $7, %ecx ; SSSE3-NEXT: movd %ecx, %xmm4 -; SSSE3-NEXT: movzbl -24(%rsp), %ecx +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx ; SSSE3-NEXT: bsrl %ecx, %ecx ; SSSE3-NEXT: cmovel %eax, %ecx ; SSSE3-NEXT: xorl $7, %ecx @@ -1390,27 +1468,43 @@ define <16 x i8> @testv16i8(<16 x i8> %in) { ; AVX-NEXT: xorl $7, %ecx ; AVX-NEXT: vpinsrb $15, %ecx, %xmm1, %xmm0 ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: testv16i8: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vpmovzxbd %xmm0, %zmm0 +; AVX512VLCD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512VLCD-NEXT: vpmovdb %zmm0, %xmm0 +; AVX512VLCD-NEXT: vpsubb {{.*}}(%rip), %xmm0, %xmm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: testv16i8: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vpmovzxbd %xmm0, %zmm0 +; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512CD-NEXT: vpmovdb %zmm0, %xmm0 +; AVX512CD-NEXT: vpsubb {{.*}}(%rip), %xmm0, %xmm0 +; AVX512CD-NEXT: retq %out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %in, i1 0) ret <16 x i8> %out } -define <16 x i8> @testv16i8u(<16 x i8> %in) { +define <16 x i8> @testv16i8u(<16 x i8> %in) nounwind { ; SSE2-LABEL: testv16i8u: ; SSE2: # BB#0: -; SSE2: pushq %rbx -; SSE2: movaps %xmm0, -16(%rsp) -; SSE2-NEXT: movzbl -1(%rsp), %eax +; SSE2-NEXT: pushq %rbx +; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp) +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax ; SSE2-NEXT: bsrl %eax, %eax ; SSE2-NEXT: xorl $7, %eax ; SSE2-NEXT: movd %eax, %xmm0 -; SSE2-NEXT: movzbl -2(%rsp), %edi -; SSE2-NEXT: movzbl -3(%rsp), %edx -; SSE2-NEXT: movzbl -4(%rsp), %r9d -; SSE2-NEXT: movzbl -5(%rsp), %eax -; SSE2-NEXT: movzbl -6(%rsp), %r10d -; SSE2-NEXT: movzbl -7(%rsp), %ecx -; SSE2-NEXT: movzbl -8(%rsp), %r8d -; SSE2-NEXT: movzbl -9(%rsp), %esi +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edi +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r9d +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r10d +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r8d +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi ; SSE2-NEXT: bsrl %esi, %esi ; SSE2-NEXT: xorl $7, %esi ; SSE2-NEXT: movd %esi, %xmm1 @@ -1418,10 +1512,10 @@ define <16 x i8> @testv16i8u(<16 x i8> %in) { ; SSE2-NEXT: bsrl %eax, %eax ; SSE2-NEXT: xorl $7, %eax ; SSE2-NEXT: movd %eax, %xmm0 -; SSE2-NEXT: movzbl -10(%rsp), %eax -; SSE2-NEXT: movzbl -11(%rsp), %esi -; SSE2-NEXT: movzbl -12(%rsp), %r11d -; SSE2-NEXT: movzbl -13(%rsp), %ebx +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r11d +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebx ; SSE2-NEXT: bsrl %ebx, %ebx ; SSE2-NEXT: xorl $7, %ebx ; SSE2-NEXT: movd %ebx, %xmm2 @@ -1437,8 +1531,8 @@ define <16 x i8> @testv16i8u(<16 x i8> %in) { ; SSE2-NEXT: bsrl %ecx, %ecx ; SSE2-NEXT: xorl $7, %ecx ; SSE2-NEXT: movd %ecx, %xmm0 -; SSE2-NEXT: movzbl -14(%rsp), %ecx -; SSE2-NEXT: movzbl -15(%rsp), %edx +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx ; SSE2-NEXT: bsrl %edx, %edx ; SSE2-NEXT: xorl $7, %edx ; SSE2-NEXT: movd %edx, %xmm1 @@ -1470,7 +1564,7 @@ define <16 x i8> @testv16i8u(<16 x i8> %in) { ; SSE2-NEXT: bsrl %r8d, %eax ; SSE2-NEXT: xorl $7, %eax ; SSE2-NEXT: movd %eax, %xmm4 -; SSE2-NEXT: movzbl -16(%rsp), %eax +; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax ; SSE2-NEXT: bsrl %eax, %eax ; SSE2-NEXT: xorl $7, %eax ; SSE2-NEXT: movd %eax, %xmm0 @@ -1483,20 +1577,20 @@ define <16 x i8> @testv16i8u(<16 x i8> %in) { ; ; SSE3-LABEL: testv16i8u: ; SSE3: # BB#0: -; SSE3: pushq %rbx -; SSE3: movaps %xmm0, -16(%rsp) -; SSE3-NEXT: movzbl -1(%rsp), %eax +; SSE3-NEXT: pushq %rbx +; SSE3-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp) +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax ; SSE3-NEXT: bsrl %eax, %eax ; SSE3-NEXT: xorl $7, %eax ; SSE3-NEXT: movd %eax, %xmm0 -; SSE3-NEXT: movzbl -2(%rsp), %edi -; SSE3-NEXT: movzbl -3(%rsp), %edx -; SSE3-NEXT: movzbl -4(%rsp), %r9d -; SSE3-NEXT: movzbl -5(%rsp), %eax -; SSE3-NEXT: movzbl -6(%rsp), %r10d -; SSE3-NEXT: movzbl -7(%rsp), %ecx -; SSE3-NEXT: movzbl -8(%rsp), %r8d -; SSE3-NEXT: movzbl -9(%rsp), %esi +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %edi +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %r9d +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %r10d +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %r8d +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi ; SSE3-NEXT: bsrl %esi, %esi ; SSE3-NEXT: xorl $7, %esi ; SSE3-NEXT: movd %esi, %xmm1 @@ -1504,10 +1598,10 @@ define <16 x i8> @testv16i8u(<16 x i8> %in) { ; SSE3-NEXT: bsrl %eax, %eax ; SSE3-NEXT: xorl $7, %eax ; SSE3-NEXT: movd %eax, %xmm0 -; SSE3-NEXT: movzbl -10(%rsp), %eax -; SSE3-NEXT: movzbl -11(%rsp), %esi -; SSE3-NEXT: movzbl -12(%rsp), %r11d -; SSE3-NEXT: movzbl -13(%rsp), %ebx +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %r11d +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebx ; SSE3-NEXT: bsrl %ebx, %ebx ; SSE3-NEXT: xorl $7, %ebx ; SSE3-NEXT: movd %ebx, %xmm2 @@ -1523,8 +1617,8 @@ define <16 x i8> @testv16i8u(<16 x i8> %in) { ; SSE3-NEXT: bsrl %ecx, %ecx ; SSE3-NEXT: xorl $7, %ecx ; SSE3-NEXT: movd %ecx, %xmm0 -; SSE3-NEXT: movzbl -14(%rsp), %ecx -; SSE3-NEXT: movzbl -15(%rsp), %edx +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx ; SSE3-NEXT: bsrl %edx, %edx ; SSE3-NEXT: xorl $7, %edx ; SSE3-NEXT: movd %edx, %xmm1 @@ -1556,7 +1650,7 @@ define <16 x i8> @testv16i8u(<16 x i8> %in) { ; SSE3-NEXT: bsrl %r8d, %eax ; SSE3-NEXT: xorl $7, %eax ; SSE3-NEXT: movd %eax, %xmm4 -; SSE3-NEXT: movzbl -16(%rsp), %eax +; SSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax ; SSE3-NEXT: bsrl %eax, %eax ; SSE3-NEXT: xorl $7, %eax ; SSE3-NEXT: movd %eax, %xmm0 @@ -1569,20 +1663,20 @@ define <16 x i8> @testv16i8u(<16 x i8> %in) { ; ; SSSE3-LABEL: testv16i8u: ; SSSE3: # BB#0: -; SSSE3: pushq %rbx -; SSSE3: movaps %xmm0, -16(%rsp) -; SSSE3-NEXT: movzbl -1(%rsp), %eax +; SSSE3-NEXT: pushq %rbx +; SSSE3-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp) +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax ; SSSE3-NEXT: bsrl %eax, %eax ; SSSE3-NEXT: xorl $7, %eax ; SSSE3-NEXT: movd %eax, %xmm0 -; SSSE3-NEXT: movzbl -2(%rsp), %edi -; SSSE3-NEXT: movzbl -3(%rsp), %edx -; SSSE3-NEXT: movzbl -4(%rsp), %r9d -; SSSE3-NEXT: movzbl -5(%rsp), %eax -; SSSE3-NEXT: movzbl -6(%rsp), %r10d -; SSSE3-NEXT: movzbl -7(%rsp), %ecx -; SSSE3-NEXT: movzbl -8(%rsp), %r8d -; SSSE3-NEXT: movzbl -9(%rsp), %esi +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %edi +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %r9d +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %r10d +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %r8d +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi ; SSSE3-NEXT: bsrl %esi, %esi ; SSSE3-NEXT: xorl $7, %esi ; SSSE3-NEXT: movd %esi, %xmm1 @@ -1590,10 +1684,10 @@ define <16 x i8> @testv16i8u(<16 x i8> %in) { ; SSSE3-NEXT: bsrl %eax, %eax ; SSSE3-NEXT: xorl $7, %eax ; SSSE3-NEXT: movd %eax, %xmm0 -; SSSE3-NEXT: movzbl -10(%rsp), %eax -; SSSE3-NEXT: movzbl -11(%rsp), %esi -; SSSE3-NEXT: movzbl -12(%rsp), %r11d -; SSSE3-NEXT: movzbl -13(%rsp), %ebx +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %r11d +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebx ; SSSE3-NEXT: bsrl %ebx, %ebx ; SSSE3-NEXT: xorl $7, %ebx ; SSSE3-NEXT: movd %ebx, %xmm2 @@ -1609,8 +1703,8 @@ define <16 x i8> @testv16i8u(<16 x i8> %in) { ; SSSE3-NEXT: bsrl %ecx, %ecx ; SSSE3-NEXT: xorl $7, %ecx ; SSSE3-NEXT: movd %ecx, %xmm0 -; SSSE3-NEXT: movzbl -14(%rsp), %ecx -; SSSE3-NEXT: movzbl -15(%rsp), %edx +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx ; SSSE3-NEXT: bsrl %edx, %edx ; SSSE3-NEXT: xorl $7, %edx ; SSSE3-NEXT: movd %edx, %xmm1 @@ -1642,7 +1736,7 @@ define <16 x i8> @testv16i8u(<16 x i8> %in) { ; SSSE3-NEXT: bsrl %r8d, %eax ; SSSE3-NEXT: xorl $7, %eax ; SSSE3-NEXT: movd %eax, %xmm4 -; SSSE3-NEXT: movzbl -16(%rsp), %eax +; SSSE3-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax ; SSSE3-NEXT: bsrl %eax, %eax ; SSSE3-NEXT: xorl $7, %eax ; SSSE3-NEXT: movd %eax, %xmm0 @@ -1789,11 +1883,27 @@ define <16 x i8> @testv16i8u(<16 x i8> %in) { ; AVX-NEXT: xorl $7, %eax ; AVX-NEXT: vpinsrb $15, %eax, %xmm1, %xmm0 ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: testv16i8u: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vpmovzxbd %xmm0, %zmm0 +; AVX512VLCD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512VLCD-NEXT: vpmovdb %zmm0, %xmm0 +; AVX512VLCD-NEXT: vpsubb {{.*}}(%rip), %xmm0, %xmm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: testv16i8u: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vpmovzxbd %xmm0, %zmm0 +; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512CD-NEXT: vpmovdb %zmm0, %xmm0 +; AVX512CD-NEXT: vpsubb {{.*}}(%rip), %xmm0, %xmm0 +; AVX512CD-NEXT: retq %out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %in, i1 -1) ret <16 x i8> %out } -define <2 x i64> @foldv2i64() { +define <2 x i64> @foldv2i64() nounwind { ; SSE-LABEL: foldv2i64: ; SSE: # BB#0: ; SSE-NEXT: movl $55, %eax @@ -1805,11 +1915,23 @@ define <2 x i64> @foldv2i64() { ; AVX-NEXT: movl $55, %eax ; AVX-NEXT: vmovq %rax, %xmm0 ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: foldv2i64: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: movl $55, %eax +; AVX512VLCD-NEXT: vmovq %rax, %xmm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: foldv2i64: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: movl $55, %eax +; AVX512CD-NEXT: vmovq %rax, %xmm0 +; AVX512CD-NEXT: retq %out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> , i1 0) ret <2 x i64> %out } -define <2 x i64> @foldv2i64u() { +define <2 x i64> @foldv2i64u() nounwind { ; SSE-LABEL: foldv2i64u: ; SSE: # BB#0: ; SSE-NEXT: movl $55, %eax @@ -1821,11 +1943,23 @@ define <2 x i64> @foldv2i64u() { ; AVX-NEXT: movl $55, %eax ; AVX-NEXT: vmovq %rax, %xmm0 ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: foldv2i64u: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: movl $55, %eax +; AVX512VLCD-NEXT: vmovq %rax, %xmm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: foldv2i64u: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: movl $55, %eax +; AVX512CD-NEXT: vmovq %rax, %xmm0 +; AVX512CD-NEXT: retq %out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> , i1 -1) ret <2 x i64> %out } -define <4 x i32> @foldv4i32() { +define <4 x i32> @foldv4i32() nounwind { ; SSE-LABEL: foldv4i32: ; SSE: # BB#0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [23,0,32,24] @@ -1835,11 +1969,21 @@ define <4 x i32> @foldv4i32() { ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [23,0,32,24] ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: foldv4i32: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vmovdqa32 {{.*#+}} xmm0 = [23,0,32,24] +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: foldv4i32: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vmovaps {{.*#+}} xmm0 = [23,0,32,24] +; AVX512CD-NEXT: retq %out = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> , i1 0) ret <4 x i32> %out } -define <4 x i32> @foldv4i32u() { +define <4 x i32> @foldv4i32u() nounwind { ; SSE-LABEL: foldv4i32u: ; SSE: # BB#0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [23,0,32,24] @@ -1849,11 +1993,21 @@ define <4 x i32> @foldv4i32u() { ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [23,0,32,24] ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: foldv4i32u: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vmovdqa32 {{.*#+}} xmm0 = [23,0,32,24] +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: foldv4i32u: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vmovaps {{.*#+}} xmm0 = [23,0,32,24] +; AVX512CD-NEXT: retq %out = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> , i1 -1) ret <4 x i32> %out } -define <8 x i16> @foldv8i16() { +define <8 x i16> @foldv8i16() nounwind { ; SSE-LABEL: foldv8i16: ; SSE: # BB#0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [7,0,16,8,16,13,11,9] @@ -1863,11 +2017,21 @@ define <8 x i16> @foldv8i16() { ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [7,0,16,8,16,13,11,9] ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: foldv8i16: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vmovdqa64 {{.*#+}} xmm0 = [7,0,16,8,16,13,11,9] +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: foldv8i16: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vmovaps {{.*#+}} xmm0 = [7,0,16,8,16,13,11,9] +; AVX512CD-NEXT: retq %out = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> , i1 0) ret <8 x i16> %out } -define <8 x i16> @foldv8i16u() { +define <8 x i16> @foldv8i16u() nounwind { ; SSE-LABEL: foldv8i16u: ; SSE: # BB#0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [7,0,16,8,16,13,11,9] @@ -1877,11 +2041,21 @@ define <8 x i16> @foldv8i16u() { ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [7,0,16,8,16,13,11,9] ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: foldv8i16u: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vmovdqa64 {{.*#+}} xmm0 = [7,0,16,8,16,13,11,9] +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: foldv8i16u: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vmovaps {{.*#+}} xmm0 = [7,0,16,8,16,13,11,9] +; AVX512CD-NEXT: retq %out = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> , i1 -1) ret <8 x i16> %out } -define <16 x i8> @foldv16i8() { +define <16 x i8> @foldv16i8() nounwind { ; SSE-LABEL: foldv16i8: ; SSE: # BB#0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2] @@ -1891,11 +2065,21 @@ define <16 x i8> @foldv16i8() { ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2] ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: foldv16i8: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vmovdqa64 {{.*#+}} xmm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2] +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: foldv16i8: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vmovaps {{.*#+}} xmm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2] +; AVX512CD-NEXT: retq %out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> , i1 0) ret <16 x i8> %out } -define <16 x i8> @foldv16i8u() { +define <16 x i8> @foldv16i8u() nounwind { ; SSE-LABEL: foldv16i8u: ; SSE: # BB#0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2] @@ -1905,6 +2089,16 @@ define <16 x i8> @foldv16i8u() { ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2] ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: foldv16i8u: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vmovdqa64 {{.*#+}} xmm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2] +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: foldv16i8u: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vmovaps {{.*#+}} xmm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2] +; AVX512CD-NEXT: retq %out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> , i1 -1) ret <16 x i8> %out } diff --git a/test/CodeGen/X86/vector-lzcnt-256.ll b/test/CodeGen/X86/vector-lzcnt-256.ll index 48abe1290528..1608bf53748d 100644 --- a/test/CodeGen/X86/vector-lzcnt-256.ll +++ b/test/CodeGen/X86/vector-lzcnt-256.ll @@ -1,9 +1,10 @@ -; RUN: llc < %s -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 -; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512cd -mattr=+avx512vl| FileCheck %s --check-prefix=AVX512VLCD --check-prefix=ALL --check-prefix=AVX512 +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512cd | FileCheck %s --check-prefix=AVX512CD --check-prefix=ALL --check-prefix=AVX512 -target triple = "x86_64-unknown-unknown" - -define <4 x i64> @testv4i64(<4 x i64> %in) { +define <4 x i64> @testv4i64(<4 x i64> %in) nounwind { ; AVX1-LABEL: testv4i64: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -61,11 +62,22 @@ define <4 x i64> @testv4i64(<4 x i64> %in) { ; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; AVX512VLCD-LABEL: testv4i64: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vplzcntq %ymm0, %ymm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: testv4i64: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vplzcntq %zmm0, %zmm0 +; AVX512CD-NEXT: retq + %out = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %in, i1 0) ret <4 x i64> %out } -define <4 x i64> @testv4i64u(<4 x i64> %in) { +define <4 x i64> @testv4i64u(<4 x i64> %in) nounwind { ; AVX1-LABEL: testv4i64u: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -113,11 +125,22 @@ define <4 x i64> @testv4i64u(<4 x i64> %in) { ; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; AVX512VLCD-LABEL: testv4i64u: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vplzcntq %ymm0, %ymm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: testv4i64u: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vplzcntq %zmm0, %zmm0 +; AVX512CD-NEXT: retq + %out = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %in, i1 -1) ret <4 x i64> %out } -define <8 x i32> @testv8i32(<8 x i32> %in) { +define <8 x i32> @testv8i32(<8 x i32> %in) nounwind { ; AVX1-LABEL: testv8i32: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -211,11 +234,22 @@ define <8 x i32> @testv8i32(<8 x i32> %in) { ; AVX2-NEXT: vpinsrd $3, %ecx, %xmm2, %xmm0 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; AVX512VLCD-LABEL: testv8i32: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vplzcntd %ymm0, %ymm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: testv8i32: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512CD-NEXT: retq + %out = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %in, i1 0) ret <8 x i32> %out } -define <8 x i32> @testv8i32u(<8 x i32> %in) { +define <8 x i32> @testv8i32u(<8 x i32> %in) nounwind { ; AVX1-LABEL: testv8i32u: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -291,11 +325,22 @@ define <8 x i32> @testv8i32u(<8 x i32> %in) { ; AVX2-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; AVX512VLCD-LABEL: testv8i32u: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vplzcntd %ymm0, %ymm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: testv8i32u: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512CD-NEXT: retq + %out = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %in, i1 -1) ret <8 x i32> %out } -define <16 x i16> @testv16i16(<16 x i16> %in) { +define <16 x i16> @testv16i16(<16 x i16> %in) nounwind { ; AVX1-LABEL: testv16i16: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -469,11 +514,27 @@ define <16 x i16> @testv16i16(<16 x i16> %in) { ; AVX2-NEXT: vpinsrw $7, %ecx, %xmm2, %xmm0 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; AVX512VLCD-LABEL: testv16i16: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vpmovzxwd %ymm0, %zmm0 +; AVX512VLCD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512VLCD-NEXT: vpmovdw %zmm0, %ymm0 +; AVX512VLCD-NEXT: vpsubw {{.*}}(%rip), %ymm0, %ymm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: testv16i16: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vpmovzxwd %ymm0, %zmm0 +; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512CD-NEXT: vpmovdw %zmm0, %ymm0 +; AVX512CD-NEXT: vpsubw {{.*}}(%rip), %ymm0, %ymm0 +; AVX512CD-NEXT: retq %out = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> %in, i1 0) ret <16 x i16> %out } -define <16 x i16> @testv16i16u(<16 x i16> %in) { +define <16 x i16> @testv16i16u(<16 x i16> %in) nounwind { ; AVX1-LABEL: testv16i16u: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -613,11 +674,27 @@ define <16 x i16> @testv16i16u(<16 x i16> %in) { ; AVX2-NEXT: vpinsrw $7, %eax, %xmm2, %xmm0 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; AVX512VLCD-LABEL: testv16i16u: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vpmovzxwd %ymm0, %zmm0 +; AVX512VLCD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512VLCD-NEXT: vpmovdw %zmm0, %ymm0 +; AVX512VLCD-NEXT: vpsubw {{.*}}(%rip), %ymm0, %ymm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: testv16i16u: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vpmovzxwd %ymm0, %zmm0 +; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512CD-NEXT: vpmovdw %zmm0, %ymm0 +; AVX512CD-NEXT: vpsubw {{.*}}(%rip), %ymm0, %ymm0 +; AVX512CD-NEXT: retq %out = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> %in, i1 -1) ret <16 x i16> %out } -define <32 x i8> @testv32i8(<32 x i8> %in) { +define <32 x i8> @testv32i8(<32 x i8> %in) nounwind { ; AVX1-LABEL: testv32i8: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -951,11 +1028,41 @@ define <32 x i8> @testv32i8(<32 x i8> %in) { ; AVX2-NEXT: vpinsrb $15, %ecx, %xmm2, %xmm0 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; AVX512VLCD-LABEL: testv32i8: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX512VLCD-NEXT: vpmovzxbd %xmm1, %zmm1 +; AVX512VLCD-NEXT: vplzcntd %zmm1, %zmm1 +; AVX512VLCD-NEXT: vpmovdb %zmm1, %xmm1 +; AVX512VLCD-NEXT: vmovdqa64 {{.*#+}} xmm2 = [24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24] +; AVX512VLCD-NEXT: vpsubb %xmm2, %xmm1, %xmm1 +; AVX512VLCD-NEXT: vpmovzxbd %xmm0, %zmm0 +; AVX512VLCD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512VLCD-NEXT: vpmovdb %zmm0, %xmm0 +; AVX512VLCD-NEXT: vpsubb %xmm2, %xmm0, %xmm0 +; AVX512VLCD-NEXT: vinserti32x4 $1, %xmm1, %ymm0, %ymm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: testv32i8: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX512CD-NEXT: vpmovzxbd %xmm1, %zmm1 +; AVX512CD-NEXT: vplzcntd %zmm1, %zmm1 +; AVX512CD-NEXT: vpmovdb %zmm1, %xmm1 +; AVX512CD-NEXT: vmovdqa {{.*#+}} xmm2 = [24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24] +; AVX512CD-NEXT: vpsubb %xmm2, %xmm1, %xmm1 +; AVX512CD-NEXT: vpmovzxbd %xmm0, %zmm0 +; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512CD-NEXT: vpmovdb %zmm0, %xmm0 +; AVX512CD-NEXT: vpsubb %xmm2, %xmm0, %xmm0 +; AVX512CD-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX512CD-NEXT: retq %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %in, i1 0) ret <32 x i8> %out } -define <32 x i8> @testv32i8u(<32 x i8> %in) { +define <32 x i8> @testv32i8u(<32 x i8> %in) nounwind { ; AVX1-LABEL: testv32i8u: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -1223,78 +1330,188 @@ define <32 x i8> @testv32i8u(<32 x i8> %in) { ; AVX2-NEXT: vpinsrb $15, %eax, %xmm2, %xmm0 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; AVX512VLCD-LABEL: testv32i8u: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX512VLCD-NEXT: vpmovzxbd %xmm1, %zmm1 +; AVX512VLCD-NEXT: vplzcntd %zmm1, %zmm1 +; AVX512VLCD-NEXT: vpmovdb %zmm1, %xmm1 +; AVX512VLCD-NEXT: vmovdqa64 {{.*#+}} xmm2 = [24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24] +; AVX512VLCD-NEXT: vpsubb %xmm2, %xmm1, %xmm1 +; AVX512VLCD-NEXT: vpmovzxbd %xmm0, %zmm0 +; AVX512VLCD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512VLCD-NEXT: vpmovdb %zmm0, %xmm0 +; AVX512VLCD-NEXT: vpsubb %xmm2, %xmm0, %xmm0 +; AVX512VLCD-NEXT: vinserti32x4 $1, %xmm1, %ymm0, %ymm0 +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: testv32i8u: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX512CD-NEXT: vpmovzxbd %xmm1, %zmm1 +; AVX512CD-NEXT: vplzcntd %zmm1, %zmm1 +; AVX512CD-NEXT: vpmovdb %zmm1, %xmm1 +; AVX512CD-NEXT: vmovdqa {{.*#+}} xmm2 = [24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24] +; AVX512CD-NEXT: vpsubb %xmm2, %xmm1, %xmm1 +; AVX512CD-NEXT: vpmovzxbd %xmm0, %zmm0 +; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512CD-NEXT: vpmovdb %zmm0, %xmm0 +; AVX512CD-NEXT: vpsubb %xmm2, %xmm0, %xmm0 +; AVX512CD-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX512CD-NEXT: retq %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %in, i1 -1) ret <32 x i8> %out } -define <4 x i64> @foldv4i64() { +define <4 x i64> @foldv4i64() nounwind { ; AVX-LABEL: foldv4i64: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [55,0,64,56] ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: foldv4i64: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vmovdqa64 {{.*#+}} ymm0 = [55,0,64,56] +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: foldv4i64: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vmovaps {{.*#+}} ymm0 = [55,0,64,56] +; AVX512CD-NEXT: retq %out = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> , i1 0) ret <4 x i64> %out } -define <4 x i64> @foldv4i64u() { +define <4 x i64> @foldv4i64u() nounwind { ; AVX-LABEL: foldv4i64u: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [55,0,64,56] ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: foldv4i64u: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vmovdqa64 {{.*#+}} ymm0 = [55,0,64,56] +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: foldv4i64u: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vmovaps {{.*#+}} ymm0 = [55,0,64,56] +; AVX512CD-NEXT: retq %out = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> , i1 -1) ret <4 x i64> %out } -define <8 x i32> @foldv8i32() { +define <8 x i32> @foldv8i32() nounwind { ; AVX-LABEL: foldv8i32: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25] ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: foldv8i32: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vmovdqa32 {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25] +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: foldv8i32: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vmovaps {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25] +; AVX512CD-NEXT: retq %out = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> , i1 0) ret <8 x i32> %out } -define <8 x i32> @foldv8i32u() { +define <8 x i32> @foldv8i32u() nounwind { ; AVX-LABEL: foldv8i32u: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25] ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: foldv8i32u: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vmovdqa32 {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25] +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: foldv8i32u: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vmovaps {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25] +; AVX512CD-NEXT: retq %out = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> , i1 -1) ret <8 x i32> %out } -define <16 x i16> @foldv16i16() { +define <16 x i16> @foldv16i16() nounwind { ; AVX-LABEL: foldv16i16: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [7,0,16,8,16,13,11,9,0,8,15,14,13,12,11,10] ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: foldv16i16: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vmovdqa64 {{.*#+}} ymm0 = [7,0,16,8,16,13,11,9,0,8,15,14,13,12,11,10] +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: foldv16i16: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vmovaps {{.*#+}} ymm0 = [7,0,16,8,16,13,11,9,0,8,15,14,13,12,11,10] +; AVX512CD-NEXT: retq %out = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> , i1 0) ret <16 x i16> %out } -define <16 x i16> @foldv16i16u() { +define <16 x i16> @foldv16i16u() nounwind { ; AVX-LABEL: foldv16i16u: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [7,0,16,8,16,13,11,9,0,8,15,14,13,12,11,10] ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: foldv16i16u: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vmovdqa64 {{.*#+}} ymm0 = [7,0,16,8,16,13,11,9,0,8,15,14,13,12,11,10] +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: foldv16i16u: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vmovaps {{.*#+}} ymm0 = [7,0,16,8,16,13,11,9,0,8,15,14,13,12,11,10] +; AVX512CD-NEXT: retq %out = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> , i1 -1) ret <16 x i16> %out } -define <32 x i8> @foldv32i8() { +define <32 x i8> @foldv32i8() nounwind { ; AVX-LABEL: foldv32i8: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2,1,0,8,8,0,0,0,0,0,0,0,0,6,5,5,1] ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: foldv32i8: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vmovdqa64 {{.*#+}} ymm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2,1,0,8,8,0,0,0,0,0,0,0,0,6,5,5,1] +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: foldv32i8: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2,1,0,8,8,0,0,0,0,0,0,0,0,6,5,5,1] +; AVX512CD-NEXT: retq %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> , i1 0) ret <32 x i8> %out } -define <32 x i8> @foldv32i8u() { +define <32 x i8> @foldv32i8u() nounwind { ; AVX-LABEL: foldv32i8u: ; AVX: # BB#0: ; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2,1,0,8,8,0,0,0,0,0,0,0,0,6,5,5,1] ; AVX-NEXT: retq +; +; AVX512VLCD-LABEL: foldv32i8u: +; AVX512VLCD: ## BB#0: +; AVX512VLCD-NEXT: vmovdqa64 {{.*#+}} ymm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2,1,0,8,8,0,0,0,0,0,0,0,0,6,5,5,1] +; AVX512VLCD-NEXT: retq +; +; AVX512CD-LABEL: foldv32i8u: +; AVX512CD: ## BB#0: +; AVX512CD-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2,1,0,8,8,0,0,0,0,0,0,0,0,6,5,5,1] +; AVX512CD-NEXT: retq %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> , i1 -1) ret <32 x i8> %out } diff --git a/test/CodeGen/X86/vector-lzcnt-512.ll b/test/CodeGen/X86/vector-lzcnt-512.ll new file mode 100644 index 000000000000..20ea86e5d439 --- /dev/null +++ b/test/CodeGen/X86/vector-lzcnt-512.ll @@ -0,0 +1,219 @@ +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512cd | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512CD +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512bw | FileCheck %s --check-prefix=AVX512BW + +define <8 x i64> @testv8i64(<8 x i64> %in) nounwind { +; ALL-LABEL: testv8i64: +; ALL: ## BB#0: +; ALL-NEXT: vplzcntq %zmm0, %zmm0 +; ALL-NEXT: retq + %out = call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> %in, i1 0) + ret <8 x i64> %out +} + +define <8 x i64> @testv8i64u(<8 x i64> %in) nounwind { +; ALL-LABEL: testv8i64u: +; ALL: ## BB#0: +; ALL-NEXT: vplzcntq %zmm0, %zmm0 +; ALL-NEXT: retq + %out = call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> %in, i1 -1) + ret <8 x i64> %out +} + +define <16 x i32> @testv16i32(<16 x i32> %in) nounwind { +; ALL-LABEL: testv16i32: +; ALL: ## BB#0: +; ALL-NEXT: vplzcntd %zmm0, %zmm0 +; ALL-NEXT: retq + %out = call <16 x i32> @llvm.ctlz.v16i32(<16 x i32> %in, i1 0) + ret <16 x i32> %out +} + +define <16 x i32> @testv16i32u(<16 x i32> %in) nounwind { +; ALL-LABEL: testv16i32u: +; ALL: ## BB#0: +; ALL-NEXT: vplzcntd %zmm0, %zmm0 +; ALL-NEXT: retq + %out = call <16 x i32> @llvm.ctlz.v16i32(<16 x i32> %in, i1 -1) + ret <16 x i32> %out +} + +define <32 x i16> @testv32i16(<32 x i16> %in) nounwind { +; ALL-LABEL: testv32i16: +; ALL: ## BB#0: +; ALL-NEXT: vpmovzxwd %ymm0, %zmm0 +; ALL-NEXT: vplzcntd %zmm0, %zmm0 +; ALL-NEXT: vpmovdw %zmm0, %ymm0 +; ALL-NEXT: vmovdqa {{.*#+}} ymm2 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] +; ALL-NEXT: vpsubw %ymm2, %ymm0, %ymm0 +; ALL-NEXT: vpmovzxwd %ymm1, %zmm1 +; ALL-NEXT: vplzcntd %zmm1, %zmm1 +; ALL-NEXT: vpmovdw %zmm1, %ymm1 +; ALL-NEXT: vpsubw %ymm2, %ymm1, %ymm1 +; ALL-NEXT: retq +; +; AVX512BW-LABEL: testv32i16: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1 +; AVX512BW-NEXT: vpmovzxwd %ymm1, %zmm1 +; AVX512BW-NEXT: vplzcntd %zmm1, %zmm1 +; AVX512BW-NEXT: vpmovdw %zmm1, %ymm1 +; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm2 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] +; AVX512BW-NEXT: vpsubw %ymm2, %ymm1, %ymm1 +; AVX512BW-NEXT: vpmovzxwd %ymm0, %zmm0 +; AVX512BW-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512BW-NEXT: vpmovdw %zmm0, %ymm0 +; AVX512BW-NEXT: vpsubw %ymm2, %ymm0, %ymm0 +; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %out = call <32 x i16> @llvm.ctlz.v32i16(<32 x i16> %in, i1 0) + ret <32 x i16> %out +} + +define <32 x i16> @testv32i16u(<32 x i16> %in) nounwind { +; ALL-LABEL: testv32i16u: +; ALL: ## BB#0: +; ALL-NEXT: vpmovzxwd %ymm0, %zmm0 +; ALL-NEXT: vplzcntd %zmm0, %zmm0 +; ALL-NEXT: vpmovdw %zmm0, %ymm0 +; ALL-NEXT: vmovdqa {{.*#+}} ymm2 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] +; ALL-NEXT: vpsubw %ymm2, %ymm0, %ymm0 +; ALL-NEXT: vpmovzxwd %ymm1, %zmm1 +; ALL-NEXT: vplzcntd %zmm1, %zmm1 +; ALL-NEXT: vpmovdw %zmm1, %ymm1 +; ALL-NEXT: vpsubw %ymm2, %ymm1, %ymm1 +; ALL-NEXT: retq +; +; AVX512BW-LABEL: testv32i16u: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1 +; AVX512BW-NEXT: vpmovzxwd %ymm1, %zmm1 +; AVX512BW-NEXT: vplzcntd %zmm1, %zmm1 +; AVX512BW-NEXT: vpmovdw %zmm1, %ymm1 +; AVX512BW-NEXT: vmovdqa {{.*#+}} ymm2 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] +; AVX512BW-NEXT: vpsubw %ymm2, %ymm1, %ymm1 +; AVX512BW-NEXT: vpmovzxwd %ymm0, %zmm0 +; AVX512BW-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512BW-NEXT: vpmovdw %zmm0, %ymm0 +; AVX512BW-NEXT: vpsubw %ymm2, %ymm0, %ymm0 +; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %out = call <32 x i16> @llvm.ctlz.v32i16(<32 x i16> %in, i1 -1) + ret <32 x i16> %out +} + +define <64 x i8> @testv64i8(<64 x i8> %in) nounwind { +; ALL-LABEL: testv64i8: +; ALL: ## BB#0: +; ALL-NEXT: vextractf128 $1, %ymm0, %xmm2 +; ALL-NEXT: vpmovzxbd %xmm2, %zmm2 +; ALL-NEXT: vplzcntd %zmm2, %zmm2 +; ALL-NEXT: vpmovdb %zmm2, %xmm2 +; ALL-NEXT: vmovdqa {{.*#+}} xmm3 = [24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24] +; ALL-NEXT: vpsubb %xmm3, %xmm2, %xmm2 +; ALL-NEXT: vpmovzxbd %xmm0, %zmm0 +; ALL-NEXT: vplzcntd %zmm0, %zmm0 +; ALL-NEXT: vpmovdb %zmm0, %xmm0 +; ALL-NEXT: vpsubb %xmm3, %xmm0, %xmm0 +; ALL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; ALL-NEXT: vextractf128 $1, %ymm1, %xmm2 +; ALL-NEXT: vpmovzxbd %xmm2, %zmm2 +; ALL-NEXT: vplzcntd %zmm2, %zmm2 +; ALL-NEXT: vpmovdb %zmm2, %xmm2 +; ALL-NEXT: vpsubb %xmm3, %xmm2, %xmm2 +; ALL-NEXT: vpmovzxbd %xmm1, %zmm1 +; ALL-NEXT: vplzcntd %zmm1, %zmm1 +; ALL-NEXT: vpmovdb %zmm1, %xmm1 +; ALL-NEXT: vpsubb %xmm3, %xmm1, %xmm1 +; ALL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; ALL-NEXT: retq +; +; AVX512BW-LABEL: testv64i8: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1 +; AVX512BW-NEXT: vextracti128 $1, %ymm1, %xmm2 +; AVX512BW-NEXT: vpmovzxbd %xmm2, %zmm2 +; AVX512BW-NEXT: vplzcntd %zmm2, %zmm2 +; AVX512BW-NEXT: vpmovdb %zmm2, %xmm2 +; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm3 = [24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24] +; AVX512BW-NEXT: vpsubb %xmm3, %xmm2, %xmm2 +; AVX512BW-NEXT: vpmovzxbd %xmm1, %zmm1 +; AVX512BW-NEXT: vplzcntd %zmm1, %zmm1 +; AVX512BW-NEXT: vpmovdb %zmm1, %xmm1 +; AVX512BW-NEXT: vpsubb %xmm3, %xmm1, %xmm1 +; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm2 +; AVX512BW-NEXT: vpmovzxbd %xmm2, %zmm2 +; AVX512BW-NEXT: vplzcntd %zmm2, %zmm2 +; AVX512BW-NEXT: vpmovdb %zmm2, %xmm2 +; AVX512BW-NEXT: vpsubb %xmm3, %xmm2, %xmm2 +; AVX512BW-NEXT: vpmovzxbd %xmm0, %zmm0 +; AVX512BW-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512BW-NEXT: vpmovdb %zmm0, %xmm0 +; AVX512BW-NEXT: vpsubb %xmm3, %xmm0, %xmm0 +; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %out = call <64 x i8> @llvm.ctlz.v64i8(<64 x i8> %in, i1 0) + ret <64 x i8> %out +} + +define <64 x i8> @testv64i8u(<64 x i8> %in) nounwind { +; ALL-LABEL: testv64i8u: +; ALL: ## BB#0: +; ALL-NEXT: vextractf128 $1, %ymm0, %xmm2 +; ALL-NEXT: vpmovzxbd %xmm2, %zmm2 +; ALL-NEXT: vplzcntd %zmm2, %zmm2 +; ALL-NEXT: vpmovdb %zmm2, %xmm2 +; ALL-NEXT: vmovdqa {{.*#+}} xmm3 = [24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24] +; ALL-NEXT: vpsubb %xmm3, %xmm2, %xmm2 +; ALL-NEXT: vpmovzxbd %xmm0, %zmm0 +; ALL-NEXT: vplzcntd %zmm0, %zmm0 +; ALL-NEXT: vpmovdb %zmm0, %xmm0 +; ALL-NEXT: vpsubb %xmm3, %xmm0, %xmm0 +; ALL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; ALL-NEXT: vextractf128 $1, %ymm1, %xmm2 +; ALL-NEXT: vpmovzxbd %xmm2, %zmm2 +; ALL-NEXT: vplzcntd %zmm2, %zmm2 +; ALL-NEXT: vpmovdb %zmm2, %xmm2 +; ALL-NEXT: vpsubb %xmm3, %xmm2, %xmm2 +; ALL-NEXT: vpmovzxbd %xmm1, %zmm1 +; ALL-NEXT: vplzcntd %zmm1, %zmm1 +; ALL-NEXT: vpmovdb %zmm1, %xmm1 +; ALL-NEXT: vpsubb %xmm3, %xmm1, %xmm1 +; ALL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; ALL-NEXT: retq +; +; AVX512BW-LABEL: testv64i8u: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1 +; AVX512BW-NEXT: vextracti128 $1, %ymm1, %xmm2 +; AVX512BW-NEXT: vpmovzxbd %xmm2, %zmm2 +; AVX512BW-NEXT: vplzcntd %zmm2, %zmm2 +; AVX512BW-NEXT: vpmovdb %zmm2, %xmm2 +; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm3 = [24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24] +; AVX512BW-NEXT: vpsubb %xmm3, %xmm2, %xmm2 +; AVX512BW-NEXT: vpmovzxbd %xmm1, %zmm1 +; AVX512BW-NEXT: vplzcntd %zmm1, %zmm1 +; AVX512BW-NEXT: vpmovdb %zmm1, %xmm1 +; AVX512BW-NEXT: vpsubb %xmm3, %xmm1, %xmm1 +; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm2 +; AVX512BW-NEXT: vpmovzxbd %xmm2, %zmm2 +; AVX512BW-NEXT: vplzcntd %zmm2, %zmm2 +; AVX512BW-NEXT: vpmovdb %zmm2, %xmm2 +; AVX512BW-NEXT: vpsubb %xmm3, %xmm2, %xmm2 +; AVX512BW-NEXT: vpmovzxbd %xmm0, %zmm0 +; AVX512BW-NEXT: vplzcntd %zmm0, %zmm0 +; AVX512BW-NEXT: vpmovdb %zmm0, %xmm0 +; AVX512BW-NEXT: vpsubb %xmm3, %xmm0, %xmm0 +; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %out = call <64 x i8> @llvm.ctlz.v64i8(<64 x i8> %in, i1 -1) + ret <64 x i8> %out +} + +declare <8 x i64> @llvm.ctlz.v8i64(<8 x i64>, i1) +declare <16 x i32> @llvm.ctlz.v16i32(<16 x i32>, i1) +declare <32 x i16> @llvm.ctlz.v32i16(<32 x i16>, i1) +declare <64 x i8> @llvm.ctlz.v64i8(<64 x i8>, i1) diff --git a/test/CodeGen/X86/vector-merge-store-fp-constants.ll b/test/CodeGen/X86/vector-merge-store-fp-constants.ll new file mode 100644 index 000000000000..a6fb32d48a7c --- /dev/null +++ b/test/CodeGen/X86/vector-merge-store-fp-constants.ll @@ -0,0 +1,35 @@ +; RUN: llc -march=x86-64 -mtriple=x86_64-unknown-unknown < %s | FileCheck -check-prefix=DEFAULTCPU -check-prefix=ALL %s +; RUN: llc -march=x86-64 -mcpu=x86-64 -mtriple=x86_64-unknown-unknown < %s | FileCheck -check-prefix=X8664CPU -check-prefix=ALL %s + + +; ALL-LABEL: {{^}}merge_8_float_zero_stores: + +; DEFAULTCPU-DAG: movq $0, ([[PTR:%[a-z]+]]) +; DEFAULTCPU-DAG: movq $0, 8([[PTR]]) +; DEFAULTCPU-DAG: movq $0, 16([[PTR]]) +; DEFAULTCPU-DAG: movq $0, 24([[PTR]]) + +; X8664CPU: xorps [[ZEROREG:%xmm[0-9]+]], [[ZEROREG]] +; X8664CPU-DAG: movups [[ZEROREG]], ([[PTR:%[a-z]+]]) +; X8664CPU-DAG: movups [[ZEROREG]], 16([[PTR:%[a-z]+]]) + +; ALL: retq +define void @merge_8_float_zero_stores(float* %ptr) { + %idx0 = getelementptr float, float* %ptr, i64 0 + %idx1 = getelementptr float, float* %ptr, i64 1 + %idx2 = getelementptr float, float* %ptr, i64 2 + %idx3 = getelementptr float, float* %ptr, i64 3 + %idx4 = getelementptr float, float* %ptr, i64 4 + %idx5 = getelementptr float, float* %ptr, i64 5 + %idx6 = getelementptr float, float* %ptr, i64 6 + %idx7 = getelementptr float, float* %ptr, i64 7 + store float 0.0, float* %idx0, align 4 + store float 0.0, float* %idx1, align 4 + store float 0.0, float* %idx2, align 4 + store float 0.0, float* %idx3, align 4 + store float 0.0, float* %idx4, align 4 + store float 0.0, float* %idx5, align 4 + store float 0.0, float* %idx6, align 4 + store float 0.0, float* %idx7, align 4 + ret void +} diff --git a/test/CodeGen/X86/vector-popcnt-128.ll b/test/CodeGen/X86/vector-popcnt-128.ll index fef445de04ab..358bd4018290 100644 --- a/test/CodeGen/X86/vector-popcnt-128.ll +++ b/test/CodeGen/X86/vector-popcnt-128.ll @@ -1,13 +1,12 @@ -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE3 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE3 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 -target triple = "x86_64-unknown-unknown" - -define <2 x i64> @testv2i64(<2 x i64> %in) { +define <2 x i64> @testv2i64(<2 x i64> %in) nounwind { ; SSE2-LABEL: testv2i64: ; SSE2: # BB#0: ; SSE2-NEXT: movdqa %xmm0, %xmm1 @@ -93,13 +92,13 @@ define <2 x i64> @testv2i64(<2 x i64> %in) { ; AVX-NEXT: vpshufb %xmm0, %xmm3, %xmm0 ; AVX-NEXT: vpaddb %xmm2, %xmm0, %xmm0 ; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpsadbw %xmm0, %xmm1, %xmm0 +; AVX-NEXT: vpsadbw %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq %out = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %in) ret <2 x i64> %out } -define <4 x i32> @testv4i32(<4 x i32> %in) { +define <4 x i32> @testv4i32(<4 x i32> %in) nounwind { ; SSE2-LABEL: testv4i32: ; SSE2: # BB#0: ; SSE2-NEXT: movdqa %xmm0, %xmm1 @@ -208,16 +207,16 @@ define <4 x i32> @testv4i32(<4 x i32> %in) { ; AVX-NEXT: vpaddb %xmm2, %xmm0, %xmm0 ; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; AVX-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] -; AVX-NEXT: vpsadbw %xmm2, %xmm1, %xmm2 +; AVX-NEXT: vpsadbw %xmm1, %xmm2, %xmm2 ; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] -; AVX-NEXT: vpsadbw %xmm0, %xmm1, %xmm0 +; AVX-NEXT: vpsadbw %xmm1, %xmm0, %xmm0 ; AVX-NEXT: vpackuswb %xmm2, %xmm0, %xmm0 ; AVX-NEXT: retq %out = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %in) ret <4 x i32> %out } -define <8 x i16> @testv8i16(<8 x i16> %in) { +define <8 x i16> @testv8i16(<8 x i16> %in) nounwind { ; SSE2-LABEL: testv8i16: ; SSE2: # BB#0: ; SSE2-NEXT: movdqa %xmm0, %xmm1 @@ -316,7 +315,7 @@ define <8 x i16> @testv8i16(<8 x i16> %in) { ret <8 x i16> %out } -define <16 x i8> @testv16i8(<16 x i8> %in) { +define <16 x i8> @testv16i8(<16 x i8> %in) nounwind { ; SSE2-LABEL: testv16i8: ; SSE2: # BB#0: ; SSE2-NEXT: movdqa %xmm0, %xmm1 @@ -400,7 +399,7 @@ define <16 x i8> @testv16i8(<16 x i8> %in) { ret <16 x i8> %out } -define <2 x i64> @foldv2i64() { +define <2 x i64> @foldv2i64() nounwind { ; SSE-LABEL: foldv2i64: ; SSE: # BB#0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [1,64] @@ -414,7 +413,7 @@ define <2 x i64> @foldv2i64() { ret <2 x i64> %out } -define <4 x i32> @foldv4i32() { +define <4 x i32> @foldv4i32() nounwind { ; SSE-LABEL: foldv4i32: ; SSE: # BB#0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [1,32,0,8] @@ -428,7 +427,7 @@ define <4 x i32> @foldv4i32() { ret <4 x i32> %out } -define <8 x i16> @foldv8i16() { +define <8 x i16> @foldv8i16() nounwind { ; SSE-LABEL: foldv8i16: ; SSE: # BB#0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [1,16,0,8,0,3,2,3] @@ -442,7 +441,7 @@ define <8 x i16> @foldv8i16() { ret <8 x i16> %out } -define <16 x i8> @foldv16i8() { +define <16 x i8> @foldv16i8() nounwind { ; SSE-LABEL: foldv16i8: ; SSE: # BB#0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [0,8,0,8,0,3,2,3,7,7,1,1,1,1,1,1] diff --git a/test/CodeGen/X86/vector-popcnt-256.ll b/test/CodeGen/X86/vector-popcnt-256.ll index 7ce4f712483a..b0e39bdf49f9 100644 --- a/test/CodeGen/X86/vector-popcnt-256.ll +++ b/test/CodeGen/X86/vector-popcnt-256.ll @@ -1,9 +1,8 @@ -; RUN: llc < %s -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 -; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 -target triple = "x86_64-unknown-unknown" - -define <4 x i64> @testv4i64(<4 x i64> %in) { +define <4 x i64> @testv4i64(<4 x i64> %in) nounwind { ; AVX1-LABEL: testv4i64: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -16,14 +15,14 @@ define <4 x i64> @testv4i64(<4 x i64> %in) { ; AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1 ; AVX1-NEXT: vpaddb %xmm3, %xmm1, %xmm1 ; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX1-NEXT: vpsadbw %xmm1, %xmm3, %xmm1 +; AVX1-NEXT: vpsadbw %xmm3, %xmm1, %xmm1 ; AVX1-NEXT: vandps %xmm2, %xmm0, %xmm5 ; AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5 ; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0 ; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 ; AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0 ; AVX1-NEXT: vpaddb %xmm5, %xmm0, %xmm0 -; AVX1-NEXT: vpsadbw %xmm0, %xmm3, %xmm0 +; AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; @@ -38,13 +37,13 @@ define <4 x i64> @testv4i64(<4 x i64> %in) { ; AVX2-NEXT: vpshufb %ymm0, %ymm3, %ymm0 ; AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: vpxor %ymm1, %ymm1, %ymm1 -; AVX2-NEXT: vpsadbw %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq %out = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %in) ret <4 x i64> %out } -define <8 x i32> @testv8i32(<8 x i32> %in) { +define <8 x i32> @testv8i32(<8 x i32> %in) nounwind { ; AVX1-LABEL: testv8i32: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -58,9 +57,9 @@ define <8 x i32> @testv8i32(<8 x i32> %in) { ; AVX1-NEXT: vpaddb %xmm3, %xmm1, %xmm1 ; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 ; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm1[2],xmm3[2],xmm1[3],xmm3[3] -; AVX1-NEXT: vpsadbw %xmm5, %xmm3, %xmm5 +; AVX1-NEXT: vpsadbw %xmm3, %xmm5, %xmm5 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] -; AVX1-NEXT: vpsadbw %xmm1, %xmm3, %xmm1 +; AVX1-NEXT: vpsadbw %xmm3, %xmm1, %xmm1 ; AVX1-NEXT: vpackuswb %xmm5, %xmm1, %xmm1 ; AVX1-NEXT: vandps %xmm2, %xmm0, %xmm5 ; AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5 @@ -69,9 +68,9 @@ define <8 x i32> @testv8i32(<8 x i32> %in) { ; AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0 ; AVX1-NEXT: vpaddb %xmm5, %xmm0, %xmm0 ; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm3[2],xmm0[3],xmm3[3] -; AVX1-NEXT: vpsadbw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpsadbw %xmm3, %xmm2, %xmm2 ; AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1] -; AVX1-NEXT: vpsadbw %xmm0, %xmm3, %xmm0 +; AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0 ; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq @@ -88,16 +87,16 @@ define <8 x i32> @testv8i32(<8 x i32> %in) { ; AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: vpxor %ymm1, %ymm1, %ymm1 ; AVX2-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] -; AVX2-NEXT: vpsadbw %ymm2, %ymm1, %ymm2 +; AVX2-NEXT: vpsadbw %ymm1, %ymm2, %ymm2 ; AVX2-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] -; AVX2-NEXT: vpsadbw %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: retq %out = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %in) ret <8 x i32> %out } -define <16 x i16> @testv16i16(<16 x i16> %in) { +define <16 x i16> @testv16i16(<16 x i16> %in) nounwind { ; AVX1-LABEL: testv16i16: ; AVX1: # BB#0: ; AVX1-NEXT: vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] @@ -142,7 +141,7 @@ define <16 x i16> @testv16i16(<16 x i16> %in) { ret <16 x i16> %out } -define <32 x i8> @testv32i8(<32 x i8> %in) { +define <32 x i8> @testv32i8(<32 x i8> %in) nounwind { ; AVX1-LABEL: testv32i8: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -178,38 +177,38 @@ define <32 x i8> @testv32i8(<32 x i8> %in) { ret <32 x i8> %out } -define <4 x i64> @foldv4i64() { -; AVX-LABEL: foldv4i64: -; AVX: # BB#0: -; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [1,64,0,8] -; AVX-NEXT: retq +define <4 x i64> @foldv4i64() nounwind { +; ALL-LABEL: foldv4i64: +; ALL: # BB#0: +; ALL-NEXT: vmovaps {{.*#+}} ymm0 = [1,64,0,8] +; ALL-NEXT: retq %out = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> ) ret <4 x i64> %out } -define <8 x i32> @foldv8i32() { -; AVX-LABEL: foldv8i32: -; AVX: # BB#0: -; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [1,32,0,8,16,3,2,3] -; AVX-NEXT: retq +define <8 x i32> @foldv8i32() nounwind { +; ALL-LABEL: foldv8i32: +; ALL: # BB#0: +; ALL-NEXT: vmovaps {{.*#+}} ymm0 = [1,32,0,8,16,3,2,3] +; ALL-NEXT: retq %out = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> ) ret <8 x i32> %out } -define <16 x i16> @foldv16i16() { -; AVX-LABEL: foldv16i16: -; AVX: # BB#0: -; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [1,16,0,8,0,3,2,3,15,7,1,1,1,1,1,1] -; AVX-NEXT: retq +define <16 x i16> @foldv16i16() nounwind { +; ALL-LABEL: foldv16i16: +; ALL: # BB#0: +; ALL-NEXT: vmovaps {{.*#+}} ymm0 = [1,16,0,8,0,3,2,3,15,7,1,1,1,1,1,1] +; ALL-NEXT: retq %out = call <16 x i16> @llvm.ctpop.v16i16(<16 x i16> ) ret <16 x i16> %out } -define <32 x i8> @foldv32i8() { -; AVX-LABEL: foldv32i8: -; AVX: # BB#0: -; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [0,8,0,8,0,3,2,3,7,7,1,1,1,1,1,1,1,1,0,0,1,2,3,4,5,6,7,8,2,2,3,7] -; AVX-NEXT: retq +define <32 x i8> @foldv32i8() nounwind { +; ALL-LABEL: foldv32i8: +; ALL: # BB#0: +; ALL-NEXT: vmovaps {{.*#+}} ymm0 = [0,8,0,8,0,3,2,3,7,7,1,1,1,1,1,1,1,1,0,0,1,2,3,4,5,6,7,8,2,2,3,7] +; ALL-NEXT: retq %out = call <32 x i8> @llvm.ctpop.v32i8(<32 x i8> ) ret <32 x i8> %out } diff --git a/test/CodeGen/X86/vector-popcnt-512.ll b/test/CodeGen/X86/vector-popcnt-512.ll new file mode 100644 index 000000000000..54b7af6830c0 --- /dev/null +++ b/test/CodeGen/X86/vector-popcnt-512.ll @@ -0,0 +1,161 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512cd | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512CD + +define <8 x i64> @testv8i64(<8 x i64> %in) nounwind { +; ALL-LABEL: testv8i64: +; ALL: ## BB#0: +; ALL-NEXT: vextracti32x4 $3, %zmm0, %xmm1 +; ALL-NEXT: vpextrq $1, %xmm1, %rax +; ALL-NEXT: popcntq %rax, %rax +; ALL-NEXT: vmovq %rax, %xmm2 +; ALL-NEXT: vmovq %xmm1, %rax +; ALL-NEXT: popcntq %rax, %rax +; ALL-NEXT: vmovq %rax, %xmm1 +; ALL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] +; ALL-NEXT: vextracti32x4 $2, %zmm0, %xmm2 +; ALL-NEXT: vpextrq $1, %xmm2, %rax +; ALL-NEXT: popcntq %rax, %rax +; ALL-NEXT: vmovq %rax, %xmm3 +; ALL-NEXT: vmovq %xmm2, %rax +; ALL-NEXT: popcntq %rax, %rax +; ALL-NEXT: vmovq %rax, %xmm2 +; ALL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0] +; ALL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1 +; ALL-NEXT: vextracti32x4 $1, %zmm0, %xmm2 +; ALL-NEXT: vpextrq $1, %xmm2, %rax +; ALL-NEXT: popcntq %rax, %rax +; ALL-NEXT: vmovq %rax, %xmm3 +; ALL-NEXT: vmovq %xmm2, %rax +; ALL-NEXT: popcntq %rax, %rax +; ALL-NEXT: vmovq %rax, %xmm2 +; ALL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0] +; ALL-NEXT: vpextrq $1, %xmm0, %rax +; ALL-NEXT: popcntq %rax, %rax +; ALL-NEXT: vmovq %rax, %xmm3 +; ALL-NEXT: vmovq %xmm0, %rax +; ALL-NEXT: popcntq %rax, %rax +; ALL-NEXT: vmovq %rax, %xmm0 +; ALL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0] +; ALL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; ALL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 +; ALL-NEXT: retq + %out = call <8 x i64> @llvm.ctpop.v8i64(<8 x i64> %in) + ret <8 x i64> %out +} + +define <16 x i32> @testv16i32(<16 x i32> %in) nounwind { +; ALL-LABEL: testv16i32: +; ALL: ## BB#0: +; ALL-NEXT: vextracti32x4 $3, %zmm0, %xmm1 +; ALL-NEXT: vpextrd $1, %xmm1, %eax +; ALL-NEXT: popcntl %eax, %eax +; ALL-NEXT: vmovd %xmm1, %ecx +; ALL-NEXT: popcntl %ecx, %ecx +; ALL-NEXT: vmovd %ecx, %xmm2 +; ALL-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2 +; ALL-NEXT: vpextrd $2, %xmm1, %eax +; ALL-NEXT: popcntl %eax, %eax +; ALL-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2 +; ALL-NEXT: vpextrd $3, %xmm1, %eax +; ALL-NEXT: popcntl %eax, %eax +; ALL-NEXT: vpinsrd $3, %eax, %xmm2, %xmm1 +; ALL-NEXT: vextracti32x4 $2, %zmm0, %xmm2 +; ALL-NEXT: vpextrd $1, %xmm2, %eax +; ALL-NEXT: popcntl %eax, %eax +; ALL-NEXT: vmovd %xmm2, %ecx +; ALL-NEXT: popcntl %ecx, %ecx +; ALL-NEXT: vmovd %ecx, %xmm3 +; ALL-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3 +; ALL-NEXT: vpextrd $2, %xmm2, %eax +; ALL-NEXT: popcntl %eax, %eax +; ALL-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3 +; ALL-NEXT: vpextrd $3, %xmm2, %eax +; ALL-NEXT: popcntl %eax, %eax +; ALL-NEXT: vpinsrd $3, %eax, %xmm3, %xmm2 +; ALL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1 +; ALL-NEXT: vextracti32x4 $1, %zmm0, %xmm2 +; ALL-NEXT: vpextrd $1, %xmm2, %eax +; ALL-NEXT: popcntl %eax, %eax +; ALL-NEXT: vmovd %xmm2, %ecx +; ALL-NEXT: popcntl %ecx, %ecx +; ALL-NEXT: vmovd %ecx, %xmm3 +; ALL-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3 +; ALL-NEXT: vpextrd $2, %xmm2, %eax +; ALL-NEXT: popcntl %eax, %eax +; ALL-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3 +; ALL-NEXT: vpextrd $3, %xmm2, %eax +; ALL-NEXT: popcntl %eax, %eax +; ALL-NEXT: vpinsrd $3, %eax, %xmm3, %xmm2 +; ALL-NEXT: vpextrd $1, %xmm0, %eax +; ALL-NEXT: popcntl %eax, %eax +; ALL-NEXT: vmovd %xmm0, %ecx +; ALL-NEXT: popcntl %ecx, %ecx +; ALL-NEXT: vmovd %ecx, %xmm3 +; ALL-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3 +; ALL-NEXT: vpextrd $2, %xmm0, %eax +; ALL-NEXT: popcntl %eax, %eax +; ALL-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3 +; ALL-NEXT: vpextrd $3, %xmm0, %eax +; ALL-NEXT: popcntl %eax, %eax +; ALL-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0 +; ALL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; ALL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 +; ALL-NEXT: retq + %out = call <16 x i32> @llvm.ctpop.v16i32(<16 x i32> %in) + ret <16 x i32> %out +} + +define <32 x i16> @testv32i16(<32 x i16> %in) nounwind { +; ALL-LABEL: testv32i16: +; ALL: ## BB#0: +; ALL-NEXT: vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; ALL-NEXT: vpand %ymm2, %ymm0, %ymm3 +; ALL-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; ALL-NEXT: vpshufb %ymm3, %ymm4, %ymm3 +; ALL-NEXT: vpsrlw $4, %ymm0, %ymm0 +; ALL-NEXT: vpand %ymm2, %ymm0, %ymm0 +; ALL-NEXT: vpshufb %ymm0, %ymm4, %ymm0 +; ALL-NEXT: vpaddb %ymm3, %ymm0, %ymm0 +; ALL-NEXT: vpsllw $8, %ymm0, %ymm3 +; ALL-NEXT: vpaddb %ymm0, %ymm3, %ymm0 +; ALL-NEXT: vpsrlw $8, %ymm0, %ymm0 +; ALL-NEXT: vpand %ymm2, %ymm1, %ymm3 +; ALL-NEXT: vpshufb %ymm3, %ymm4, %ymm3 +; ALL-NEXT: vpsrlw $4, %ymm1, %ymm1 +; ALL-NEXT: vpand %ymm2, %ymm1, %ymm1 +; ALL-NEXT: vpshufb %ymm1, %ymm4, %ymm1 +; ALL-NEXT: vpaddb %ymm3, %ymm1, %ymm1 +; ALL-NEXT: vpsllw $8, %ymm1, %ymm2 +; ALL-NEXT: vpaddb %ymm1, %ymm2, %ymm1 +; ALL-NEXT: vpsrlw $8, %ymm1, %ymm1 +; ALL-NEXT: retq + %out = call <32 x i16> @llvm.ctpop.v32i16(<32 x i16> %in) + ret <32 x i16> %out +} + +define <64 x i8> @testv64i8(<64 x i8> %in) nounwind { +; ALL-LABEL: testv64i8: +; ALL: ## BB#0: +; ALL-NEXT: vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; ALL-NEXT: vpand %ymm2, %ymm0, %ymm3 +; ALL-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; ALL-NEXT: vpshufb %ymm3, %ymm4, %ymm3 +; ALL-NEXT: vpsrlw $4, %ymm0, %ymm0 +; ALL-NEXT: vpand %ymm2, %ymm0, %ymm0 +; ALL-NEXT: vpshufb %ymm0, %ymm4, %ymm0 +; ALL-NEXT: vpaddb %ymm3, %ymm0, %ymm0 +; ALL-NEXT: vpand %ymm2, %ymm1, %ymm3 +; ALL-NEXT: vpshufb %ymm3, %ymm4, %ymm3 +; ALL-NEXT: vpsrlw $4, %ymm1, %ymm1 +; ALL-NEXT: vpand %ymm2, %ymm1, %ymm1 +; ALL-NEXT: vpshufb %ymm1, %ymm4, %ymm1 +; ALL-NEXT: vpaddb %ymm3, %ymm1, %ymm1 +; ALL-NEXT: retq + %out = call <64 x i8> @llvm.ctpop.v64i8(<64 x i8> %in) + ret <64 x i8> %out +} + +declare <8 x i64> @llvm.ctpop.v8i64(<8 x i64>) +declare <16 x i32> @llvm.ctpop.v16i32(<16 x i32>) +declare <32 x i16> @llvm.ctpop.v32i16(<32 x i16>) +declare <64 x i8> @llvm.ctpop.v64i8(<64 x i8>) diff --git a/test/CodeGen/X86/vector-rotate-128.ll b/test/CodeGen/X86/vector-rotate-128.ll new file mode 100644 index 000000000000..4ad4aa46c5a0 --- /dev/null +++ b/test/CodeGen/X86/vector-rotate-128.ll @@ -0,0 +1,1595 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX2 +; +; Just one 32-bit run to make sure we do reasonable things for i64 rotates. +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=X32-SSE --check-prefix=X32-SSE2 + +; +; Variable Rotates +; + +define <2 x i64> @var_rotate_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { +; SSE2-LABEL: var_rotate_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [64,64] +; SSE2-NEXT: psubq %xmm1, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1] +; SSE2-NEXT: movdqa %xmm0, %xmm4 +; SSE2-NEXT: psllq %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: psllq %xmm1, %xmm3 +; SSE2-NEXT: movsd {{.*#+}} xmm4 = xmm3[0],xmm4[1] +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1] +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: psrlq %xmm3, %xmm1 +; SSE2-NEXT: psrlq %xmm2, %xmm0 +; SSE2-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1] +; SSE2-NEXT: orpd %xmm4, %xmm1 +; SSE2-NEXT: movapd %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: var_rotate_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [64,64] +; SSE41-NEXT: psubq %xmm1, %xmm2 +; SSE41-NEXT: movdqa %xmm0, %xmm3 +; SSE41-NEXT: psllq %xmm1, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] +; SSE41-NEXT: movdqa %xmm0, %xmm4 +; SSE41-NEXT: psllq %xmm1, %xmm4 +; SSE41-NEXT: pblendw {{.*#+}} xmm4 = xmm3[0,1,2,3],xmm4[4,5,6,7] +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: psrlq %xmm2, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1] +; SSE41-NEXT: psrlq %xmm2, %xmm0 +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7] +; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: retq +; +; AVX1-LABEL: var_rotate_v2i64: +; AVX1: # BB#0: +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [64,64] +; AVX1-NEXT: vpsubq %xmm1, %xmm2, %xmm2 +; AVX1-NEXT: vpsllq %xmm1, %xmm0, %xmm3 +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] +; AVX1-NEXT: vpsllq %xmm1, %xmm0, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7] +; AVX1-NEXT: vpsrlq %xmm2, %xmm0, %xmm3 +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1] +; AVX1-NEXT: vpsrlq %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7] +; AVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: var_rotate_v2i64: +; AVX2: # BB#0: +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [64,64] +; AVX2-NEXT: vpsubq %xmm1, %xmm2, %xmm2 +; AVX2-NEXT: vpsllvq %xmm1, %xmm0, %xmm1 +; AVX2-NEXT: vpsrlvq %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: retq +; +; XOP-LABEL: var_rotate_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vprotq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; X32-SSE-LABEL: var_rotate_v2i64: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [64,0,64,0] +; X32-SSE-NEXT: psubq %xmm1, %xmm2 +; X32-SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1] +; X32-SSE-NEXT: movdqa %xmm0, %xmm4 +; X32-SSE-NEXT: psllq %xmm3, %xmm4 +; X32-SSE-NEXT: movq {{.*#+}} xmm1 = xmm1[0],zero +; X32-SSE-NEXT: movdqa %xmm0, %xmm3 +; X32-SSE-NEXT: psllq %xmm1, %xmm3 +; X32-SSE-NEXT: movsd {{.*#+}} xmm4 = xmm3[0],xmm4[1] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1] +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psrlq %xmm3, %xmm1 +; X32-SSE-NEXT: movq {{.*#+}} xmm2 = xmm2[0],zero +; X32-SSE-NEXT: psrlq %xmm2, %xmm0 +; X32-SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1] +; X32-SSE-NEXT: orpd %xmm4, %xmm1 +; X32-SSE-NEXT: movapd %xmm1, %xmm0 +; X32-SSE-NEXT: retl + %b64 = sub <2 x i64> , %b + %shl = shl <2 x i64> %a, %b + %lshr = lshr <2 x i64> %a, %b64 + %or = or <2 x i64> %shl, %lshr + ret <2 x i64> %or +} + +define <4 x i32> @var_rotate_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { +; SSE2-LABEL: var_rotate_v4i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [32,32,32,32] +; SSE2-NEXT: psubd %xmm1, %xmm2 +; SSE2-NEXT: pslld $23, %xmm1 +; SSE2-NEXT: paddd {{.*}}(%rip), %xmm1 +; SSE2-NEXT: cvttps2dq %xmm1, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3] +; SSE2-NEXT: pmuludq %xmm0, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] +; SSE2-NEXT: pmuludq %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3] +; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] +; SSE2-NEXT: movdqa %xmm2, %xmm3 +; SSE2-NEXT: psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE2-NEXT: movdqa %xmm0, %xmm4 +; SSE2-NEXT: psrld %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm2, %xmm3 +; SSE2-NEXT: psrlq $32, %xmm3 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: psrld %xmm3, %xmm5 +; SSE2-NEXT: movsd {{.*#+}} xmm4 = xmm5[0],xmm4[1] +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,3,2,3] +; SSE2-NEXT: pxor %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm2, %xmm5 +; SSE2-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3] +; SSE2-NEXT: movdqa %xmm0, %xmm6 +; SSE2-NEXT: psrld %xmm5, %xmm6 +; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1] +; SSE2-NEXT: psrld %xmm2, %xmm0 +; SSE2-NEXT: movsd {{.*#+}} xmm6 = xmm0[0],xmm6[1] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3] +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1] +; SSE2-NEXT: por %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: var_rotate_v4i32: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [32,32,32,32] +; SSE41-NEXT: psubd %xmm1, %xmm2 +; SSE41-NEXT: pslld $23, %xmm1 +; SSE41-NEXT: paddd {{.*}}(%rip), %xmm1 +; SSE41-NEXT: cvttps2dq %xmm1, %xmm1 +; SSE41-NEXT: pmulld %xmm0, %xmm1 +; SSE41-NEXT: movdqa %xmm2, %xmm3 +; SSE41-NEXT: psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE41-NEXT: movdqa %xmm0, %xmm4 +; SSE41-NEXT: psrld %xmm3, %xmm4 +; SSE41-NEXT: movdqa %xmm2, %xmm3 +; SSE41-NEXT: psrlq $32, %xmm3 +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: psrld %xmm3, %xmm5 +; SSE41-NEXT: pblendw {{.*#+}} xmm5 = xmm5[0,1,2,3],xmm4[4,5,6,7] +; SSE41-NEXT: pxor %xmm3, %xmm3 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero +; SSE41-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3] +; SSE41-NEXT: movdqa %xmm0, %xmm3 +; SSE41-NEXT: psrld %xmm2, %xmm3 +; SSE41-NEXT: psrld %xmm4, %xmm0 +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7] +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm5[2,3],xmm0[4,5],xmm5[6,7] +; SSE41-NEXT: por %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; AVX1-LABEL: var_rotate_v4i32: +; AVX1: # BB#0: +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [32,32,32,32] +; AVX1-NEXT: vpsubd %xmm1, %xmm2, %xmm2 +; AVX1-NEXT: vpslld $23, %xmm1, %xmm1 +; AVX1-NEXT: vpaddd {{.*}}(%rip), %xmm1, %xmm1 +; AVX1-NEXT: vcvttps2dq %xmm1, %xmm1 +; AVX1-NEXT: vpmulld %xmm0, %xmm1, %xmm1 +; AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpsrld %xmm3, %xmm0, %xmm3 +; AVX1-NEXT: vpsrlq $32, %xmm2, %xmm4 +; AVX1-NEXT: vpsrld %xmm4, %xmm0, %xmm4 +; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7] +; AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4 +; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm2[2],xmm4[2],xmm2[3],xmm4[3] +; AVX1-NEXT: vpsrld %xmm4, %xmm0, %xmm4 +; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero +; AVX1-NEXT: vpsrld %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4,5,6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7] +; AVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: var_rotate_v4i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %xmm2 +; AVX2-NEXT: vpsubd %xmm1, %xmm2, %xmm2 +; AVX2-NEXT: vpsllvd %xmm1, %xmm0, %xmm1 +; AVX2-NEXT: vpsrlvd %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: retq +; +; XOP-LABEL: var_rotate_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vprotd %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; X32-SSE-LABEL: var_rotate_v4i32: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [32,32,32,32] +; X32-SSE-NEXT: psubd %xmm1, %xmm2 +; X32-SSE-NEXT: pslld $23, %xmm1 +; X32-SSE-NEXT: paddd .LCPI1_1, %xmm1 +; X32-SSE-NEXT: cvttps2dq %xmm1, %xmm1 +; X32-SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3] +; X32-SSE-NEXT: pmuludq %xmm0, %xmm1 +; X32-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] +; X32-SSE-NEXT: pmuludq %xmm3, %xmm4 +; X32-SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3] +; X32-SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] +; X32-SSE-NEXT: movdqa %xmm2, %xmm3 +; X32-SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; X32-SSE-NEXT: movdqa %xmm0, %xmm4 +; X32-SSE-NEXT: psrld %xmm3, %xmm4 +; X32-SSE-NEXT: movdqa %xmm2, %xmm3 +; X32-SSE-NEXT: psrlq $32, %xmm3 +; X32-SSE-NEXT: movdqa %xmm0, %xmm5 +; X32-SSE-NEXT: psrld %xmm3, %xmm5 +; X32-SSE-NEXT: movsd {{.*#+}} xmm4 = xmm5[0],xmm4[1] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,3,2,3] +; X32-SSE-NEXT: pxor %xmm4, %xmm4 +; X32-SSE-NEXT: movdqa %xmm2, %xmm5 +; X32-SSE-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3] +; X32-SSE-NEXT: movdqa %xmm0, %xmm6 +; X32-SSE-NEXT: psrld %xmm5, %xmm6 +; X32-SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1] +; X32-SSE-NEXT: psrld %xmm2, %xmm0 +; X32-SSE-NEXT: movsd {{.*#+}} xmm6 = xmm0[0],xmm6[1] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3] +; X32-SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1] +; X32-SSE-NEXT: por %xmm1, %xmm0 +; X32-SSE-NEXT: retl + %b32 = sub <4 x i32> , %b + %shl = shl <4 x i32> %a, %b + %lshr = lshr <4 x i32> %a, %b32 + %or = or <4 x i32> %shl, %lshr + ret <4 x i32> %or +} + +define <8 x i16> @var_rotate_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { +; SSE2-LABEL: var_rotate_v8i16: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16] +; SSE2-NEXT: psubw %xmm1, %xmm3 +; SSE2-NEXT: psllw $12, %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: psraw $15, %xmm2 +; SSE2-NEXT: movdqa %xmm0, %xmm4 +; SSE2-NEXT: psllw $8, %xmm4 +; SSE2-NEXT: pand %xmm2, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm4, %xmm2 +; SSE2-NEXT: paddw %xmm1, %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: psraw $15, %xmm4 +; SSE2-NEXT: movdqa %xmm4, %xmm5 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: psllw $4, %xmm2 +; SSE2-NEXT: pand %xmm4, %xmm2 +; SSE2-NEXT: por %xmm5, %xmm2 +; SSE2-NEXT: paddw %xmm1, %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: psraw $15, %xmm4 +; SSE2-NEXT: movdqa %xmm4, %xmm5 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: psllw $2, %xmm2 +; SSE2-NEXT: pand %xmm4, %xmm2 +; SSE2-NEXT: por %xmm5, %xmm2 +; SSE2-NEXT: paddw %xmm1, %xmm1 +; SSE2-NEXT: psraw $15, %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: psllw $1, %xmm2 +; SSE2-NEXT: pand %xmm1, %xmm2 +; SSE2-NEXT: psllw $12, %xmm3 +; SSE2-NEXT: movdqa %xmm3, %xmm1 +; SSE2-NEXT: psraw $15, %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: psrlw $8, %xmm0 +; SSE2-NEXT: pand %xmm1, %xmm0 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: paddw %xmm3, %xmm3 +; SSE2-NEXT: movdqa %xmm3, %xmm1 +; SSE2-NEXT: psraw $15, %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: psrlw $4, %xmm0 +; SSE2-NEXT: pand %xmm1, %xmm0 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: paddw %xmm3, %xmm3 +; SSE2-NEXT: movdqa %xmm3, %xmm1 +; SSE2-NEXT: psraw $15, %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: psrlw $2, %xmm0 +; SSE2-NEXT: pand %xmm1, %xmm0 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: paddw %xmm3, %xmm3 +; SSE2-NEXT: psraw $15, %xmm3 +; SSE2-NEXT: movdqa %xmm3, %xmm1 +; SSE2-NEXT: pandn %xmm0, %xmm1 +; SSE2-NEXT: psrlw $1, %xmm0 +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: por %xmm1, %xmm0 +; SSE2-NEXT: por %xmm4, %xmm0 +; SSE2-NEXT: por %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: var_rotate_v8i16: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm3 +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [16,16,16,16,16,16,16,16] +; SSE41-NEXT: psubw %xmm1, %xmm2 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: psllw $12, %xmm0 +; SSE41-NEXT: psllw $4, %xmm1 +; SSE41-NEXT: por %xmm0, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm4 +; SSE41-NEXT: paddw %xmm4, %xmm4 +; SSE41-NEXT: movdqa %xmm3, %xmm6 +; SSE41-NEXT: psllw $8, %xmm6 +; SSE41-NEXT: movdqa %xmm3, %xmm5 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pblendvb %xmm6, %xmm5 +; SSE41-NEXT: movdqa %xmm5, %xmm1 +; SSE41-NEXT: psllw $4, %xmm1 +; SSE41-NEXT: movdqa %xmm4, %xmm0 +; SSE41-NEXT: pblendvb %xmm1, %xmm5 +; SSE41-NEXT: movdqa %xmm5, %xmm1 +; SSE41-NEXT: psllw $2, %xmm1 +; SSE41-NEXT: paddw %xmm4, %xmm4 +; SSE41-NEXT: movdqa %xmm4, %xmm0 +; SSE41-NEXT: pblendvb %xmm1, %xmm5 +; SSE41-NEXT: movdqa %xmm5, %xmm1 +; SSE41-NEXT: psllw $1, %xmm1 +; SSE41-NEXT: paddw %xmm4, %xmm4 +; SSE41-NEXT: movdqa %xmm4, %xmm0 +; SSE41-NEXT: pblendvb %xmm1, %xmm5 +; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: psllw $12, %xmm0 +; SSE41-NEXT: psllw $4, %xmm2 +; SSE41-NEXT: por %xmm0, %xmm2 +; SSE41-NEXT: movdqa %xmm2, %xmm1 +; SSE41-NEXT: paddw %xmm1, %xmm1 +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: psrlw $8, %xmm4 +; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: pblendvb %xmm4, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm2 +; SSE41-NEXT: psrlw $4, %xmm2 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pblendvb %xmm2, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm2 +; SSE41-NEXT: psrlw $2, %xmm2 +; SSE41-NEXT: paddw %xmm1, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pblendvb %xmm2, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm2 +; SSE41-NEXT: psrlw $1, %xmm2 +; SSE41-NEXT: paddw %xmm1, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pblendvb %xmm2, %xmm3 +; SSE41-NEXT: por %xmm5, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm0 +; SSE41-NEXT: retq +; +; AVX1-LABEL: var_rotate_v8i16: +; AVX1: # BB#0: +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [16,16,16,16,16,16,16,16] +; AVX1-NEXT: vpsubw %xmm1, %xmm2, %xmm2 +; AVX1-NEXT: vpsllw $12, %xmm1, %xmm3 +; AVX1-NEXT: vpsllw $4, %xmm1, %xmm1 +; AVX1-NEXT: vpor %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpaddw %xmm1, %xmm1, %xmm3 +; AVX1-NEXT: vpsllw $8, %xmm0, %xmm4 +; AVX1-NEXT: vpblendvb %xmm1, %xmm4, %xmm0, %xmm1 +; AVX1-NEXT: vpsllw $4, %xmm1, %xmm4 +; AVX1-NEXT: vpblendvb %xmm3, %xmm4, %xmm1, %xmm1 +; AVX1-NEXT: vpsllw $2, %xmm1, %xmm4 +; AVX1-NEXT: vpaddw %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpblendvb %xmm3, %xmm4, %xmm1, %xmm1 +; AVX1-NEXT: vpsllw $1, %xmm1, %xmm4 +; AVX1-NEXT: vpaddw %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpblendvb %xmm3, %xmm4, %xmm1, %xmm1 +; AVX1-NEXT: vpsllw $12, %xmm2, %xmm3 +; AVX1-NEXT: vpsllw $4, %xmm2, %xmm2 +; AVX1-NEXT: vpor %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpaddw %xmm2, %xmm2, %xmm3 +; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm4 +; AVX1-NEXT: vpblendvb %xmm2, %xmm4, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm2 +; AVX1-NEXT: vpblendvb %xmm3, %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $2, %xmm0, %xmm2 +; AVX1-NEXT: vpaddw %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpblendvb %xmm3, %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $1, %xmm0, %xmm2 +; AVX1-NEXT: vpaddw %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpblendvb %xmm3, %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: var_rotate_v8i16: +; AVX2: # BB#0: +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [16,16,16,16,16,16,16,16] +; AVX2-NEXT: vpsubw %xmm1, %xmm2, %xmm2 +; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero +; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX2-NEXT: vpsllvd %ymm1, %ymm0, %ymm1 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128,0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128] +; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3] +; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero +; AVX2-NEXT: vpsrlvd %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpshufb %ymm3, %ymm0, %ymm0 +; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] +; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; XOP-LABEL: var_rotate_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vprotw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; X32-SSE-LABEL: var_rotate_v8i16: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16] +; X32-SSE-NEXT: psubw %xmm1, %xmm3 +; X32-SSE-NEXT: psllw $12, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: psraw $15, %xmm2 +; X32-SSE-NEXT: movdqa %xmm0, %xmm4 +; X32-SSE-NEXT: psllw $8, %xmm4 +; X32-SSE-NEXT: pand %xmm2, %xmm4 +; X32-SSE-NEXT: pandn %xmm0, %xmm2 +; X32-SSE-NEXT: por %xmm4, %xmm2 +; X32-SSE-NEXT: paddw %xmm1, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm4 +; X32-SSE-NEXT: psraw $15, %xmm4 +; X32-SSE-NEXT: movdqa %xmm4, %xmm5 +; X32-SSE-NEXT: pandn %xmm2, %xmm5 +; X32-SSE-NEXT: psllw $4, %xmm2 +; X32-SSE-NEXT: pand %xmm4, %xmm2 +; X32-SSE-NEXT: por %xmm5, %xmm2 +; X32-SSE-NEXT: paddw %xmm1, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm4 +; X32-SSE-NEXT: psraw $15, %xmm4 +; X32-SSE-NEXT: movdqa %xmm4, %xmm5 +; X32-SSE-NEXT: pandn %xmm2, %xmm5 +; X32-SSE-NEXT: psllw $2, %xmm2 +; X32-SSE-NEXT: pand %xmm4, %xmm2 +; X32-SSE-NEXT: por %xmm5, %xmm2 +; X32-SSE-NEXT: paddw %xmm1, %xmm1 +; X32-SSE-NEXT: psraw $15, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm4 +; X32-SSE-NEXT: pandn %xmm2, %xmm4 +; X32-SSE-NEXT: psllw $1, %xmm2 +; X32-SSE-NEXT: pand %xmm1, %xmm2 +; X32-SSE-NEXT: psllw $12, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm1 +; X32-SSE-NEXT: psraw $15, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm5 +; X32-SSE-NEXT: pandn %xmm0, %xmm5 +; X32-SSE-NEXT: psrlw $8, %xmm0 +; X32-SSE-NEXT: pand %xmm1, %xmm0 +; X32-SSE-NEXT: por %xmm5, %xmm0 +; X32-SSE-NEXT: paddw %xmm3, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm1 +; X32-SSE-NEXT: psraw $15, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm5 +; X32-SSE-NEXT: pandn %xmm0, %xmm5 +; X32-SSE-NEXT: psrlw $4, %xmm0 +; X32-SSE-NEXT: pand %xmm1, %xmm0 +; X32-SSE-NEXT: por %xmm5, %xmm0 +; X32-SSE-NEXT: paddw %xmm3, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm1 +; X32-SSE-NEXT: psraw $15, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm5 +; X32-SSE-NEXT: pandn %xmm0, %xmm5 +; X32-SSE-NEXT: psrlw $2, %xmm0 +; X32-SSE-NEXT: pand %xmm1, %xmm0 +; X32-SSE-NEXT: por %xmm5, %xmm0 +; X32-SSE-NEXT: paddw %xmm3, %xmm3 +; X32-SSE-NEXT: psraw $15, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm1 +; X32-SSE-NEXT: pandn %xmm0, %xmm1 +; X32-SSE-NEXT: psrlw $1, %xmm0 +; X32-SSE-NEXT: pand %xmm3, %xmm0 +; X32-SSE-NEXT: por %xmm1, %xmm0 +; X32-SSE-NEXT: por %xmm4, %xmm0 +; X32-SSE-NEXT: por %xmm2, %xmm0 +; X32-SSE-NEXT: retl + %b16 = sub <8 x i16> , %b + %shl = shl <8 x i16> %a, %b + %lshr = lshr <8 x i16> %a, %b16 + %or = or <8 x i16> %shl, %lshr + ret <8 x i16> %or +} + +define <16 x i8> @var_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { +; SSE2-LABEL: var_rotate_v16i8: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; SSE2-NEXT: psubb %xmm1, %xmm4 +; SSE2-NEXT: psllw $5, %xmm1 +; SSE2-NEXT: pxor %xmm3, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: psllw $4, %xmm5 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm5 +; SSE2-NEXT: pand %xmm2, %xmm5 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm5, %xmm2 +; SSE2-NEXT: paddb %xmm1, %xmm1 +; SSE2-NEXT: pxor %xmm5, %xmm5 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm6 +; SSE2-NEXT: pandn %xmm2, %xmm6 +; SSE2-NEXT: psllw $2, %xmm2 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm2 +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: por %xmm6, %xmm2 +; SSE2-NEXT: paddb %xmm1, %xmm1 +; SSE2-NEXT: pxor %xmm5, %xmm5 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: pandn %xmm2, %xmm1 +; SSE2-NEXT: paddb %xmm2, %xmm2 +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: psllw $5, %xmm4 +; SSE2-NEXT: pxor %xmm5, %xmm5 +; SSE2-NEXT: pcmpgtb %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm6 +; SSE2-NEXT: pandn %xmm0, %xmm6 +; SSE2-NEXT: psrlw $4, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: por %xmm6, %xmm0 +; SSE2-NEXT: paddb %xmm4, %xmm4 +; SSE2-NEXT: pxor %xmm5, %xmm5 +; SSE2-NEXT: pcmpgtb %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm6 +; SSE2-NEXT: pandn %xmm0, %xmm6 +; SSE2-NEXT: psrlw $2, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: por %xmm6, %xmm0 +; SSE2-NEXT: paddb %xmm4, %xmm4 +; SSE2-NEXT: pcmpgtb %xmm4, %xmm3 +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm4 +; SSE2-NEXT: psrlw $1, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: por %xmm4, %xmm0 +; SSE2-NEXT: por %xmm1, %xmm0 +; SSE2-NEXT: por %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: var_rotate_v16i8: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm1, %xmm3 +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; SSE41-NEXT: psubb %xmm3, %xmm2 +; SSE41-NEXT: psllw $5, %xmm3 +; SSE41-NEXT: movdqa %xmm1, %xmm5 +; SSE41-NEXT: psllw $4, %xmm5 +; SSE41-NEXT: pand {{.*}}(%rip), %xmm5 +; SSE41-NEXT: movdqa %xmm1, %xmm4 +; SSE41-NEXT: movdqa %xmm3, %xmm0 +; SSE41-NEXT: pblendvb %xmm5, %xmm4 +; SSE41-NEXT: movdqa %xmm4, %xmm5 +; SSE41-NEXT: psllw $2, %xmm5 +; SSE41-NEXT: pand {{.*}}(%rip), %xmm5 +; SSE41-NEXT: paddb %xmm3, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm0 +; SSE41-NEXT: pblendvb %xmm5, %xmm4 +; SSE41-NEXT: movdqa %xmm4, %xmm5 +; SSE41-NEXT: paddb %xmm5, %xmm5 +; SSE41-NEXT: paddb %xmm3, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm0 +; SSE41-NEXT: pblendvb %xmm5, %xmm4 +; SSE41-NEXT: psllw $5, %xmm2 +; SSE41-NEXT: movdqa %xmm2, %xmm3 +; SSE41-NEXT: paddb %xmm3, %xmm3 +; SSE41-NEXT: movdqa %xmm1, %xmm5 +; SSE41-NEXT: psrlw $4, %xmm5 +; SSE41-NEXT: pand {{.*}}(%rip), %xmm5 +; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: pblendvb %xmm5, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm2 +; SSE41-NEXT: psrlw $2, %xmm2 +; SSE41-NEXT: pand {{.*}}(%rip), %xmm2 +; SSE41-NEXT: movdqa %xmm3, %xmm0 +; SSE41-NEXT: pblendvb %xmm2, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm2 +; SSE41-NEXT: psrlw $1, %xmm2 +; SSE41-NEXT: pand {{.*}}(%rip), %xmm2 +; SSE41-NEXT: paddb %xmm3, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm0 +; SSE41-NEXT: pblendvb %xmm2, %xmm1 +; SSE41-NEXT: por %xmm4, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: var_rotate_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX-NEXT: vpsubb %xmm1, %xmm2, %xmm2 +; AVX-NEXT: vpsllw $5, %xmm1, %xmm1 +; AVX-NEXT: vpsllw $4, %xmm0, %xmm3 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm3, %xmm3 +; AVX-NEXT: vpblendvb %xmm1, %xmm3, %xmm0, %xmm3 +; AVX-NEXT: vpsllw $2, %xmm3, %xmm4 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm4, %xmm4 +; AVX-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpblendvb %xmm1, %xmm4, %xmm3, %xmm3 +; AVX-NEXT: vpaddb %xmm3, %xmm3, %xmm4 +; AVX-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpblendvb %xmm1, %xmm4, %xmm3, %xmm1 +; AVX-NEXT: vpsllw $5, %xmm2, %xmm2 +; AVX-NEXT: vpaddb %xmm2, %xmm2, %xmm3 +; AVX-NEXT: vpsrlw $4, %xmm0, %xmm4 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm4, %xmm4 +; AVX-NEXT: vpblendvb %xmm2, %xmm4, %xmm0, %xmm0 +; AVX-NEXT: vpsrlw $2, %xmm0, %xmm2 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX-NEXT: vpblendvb %xmm3, %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpsrlw $1, %xmm0, %xmm2 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX-NEXT: vpaddb %xmm3, %xmm3, %xmm3 +; AVX-NEXT: vpblendvb %xmm3, %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: var_rotate_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vprotb %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; X32-SSE-LABEL: var_rotate_v16i8: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; X32-SSE-NEXT: psubb %xmm1, %xmm4 +; X32-SSE-NEXT: psllw $5, %xmm1 +; X32-SSE-NEXT: pxor %xmm3, %xmm3 +; X32-SSE-NEXT: pxor %xmm2, %xmm2 +; X32-SSE-NEXT: pcmpgtb %xmm1, %xmm2 +; X32-SSE-NEXT: movdqa %xmm0, %xmm5 +; X32-SSE-NEXT: psllw $4, %xmm5 +; X32-SSE-NEXT: pand .LCPI3_1, %xmm5 +; X32-SSE-NEXT: pand %xmm2, %xmm5 +; X32-SSE-NEXT: pandn %xmm0, %xmm2 +; X32-SSE-NEXT: por %xmm5, %xmm2 +; X32-SSE-NEXT: paddb %xmm1, %xmm1 +; X32-SSE-NEXT: pxor %xmm5, %xmm5 +; X32-SSE-NEXT: pcmpgtb %xmm1, %xmm5 +; X32-SSE-NEXT: movdqa %xmm5, %xmm6 +; X32-SSE-NEXT: pandn %xmm2, %xmm6 +; X32-SSE-NEXT: psllw $2, %xmm2 +; X32-SSE-NEXT: pand .LCPI3_2, %xmm2 +; X32-SSE-NEXT: pand %xmm5, %xmm2 +; X32-SSE-NEXT: por %xmm6, %xmm2 +; X32-SSE-NEXT: paddb %xmm1, %xmm1 +; X32-SSE-NEXT: pxor %xmm5, %xmm5 +; X32-SSE-NEXT: pcmpgtb %xmm1, %xmm5 +; X32-SSE-NEXT: movdqa %xmm5, %xmm1 +; X32-SSE-NEXT: pandn %xmm2, %xmm1 +; X32-SSE-NEXT: paddb %xmm2, %xmm2 +; X32-SSE-NEXT: pand %xmm5, %xmm2 +; X32-SSE-NEXT: psllw $5, %xmm4 +; X32-SSE-NEXT: pxor %xmm5, %xmm5 +; X32-SSE-NEXT: pcmpgtb %xmm4, %xmm5 +; X32-SSE-NEXT: movdqa %xmm5, %xmm6 +; X32-SSE-NEXT: pandn %xmm0, %xmm6 +; X32-SSE-NEXT: psrlw $4, %xmm0 +; X32-SSE-NEXT: pand .LCPI3_3, %xmm0 +; X32-SSE-NEXT: pand %xmm5, %xmm0 +; X32-SSE-NEXT: por %xmm6, %xmm0 +; X32-SSE-NEXT: paddb %xmm4, %xmm4 +; X32-SSE-NEXT: pxor %xmm5, %xmm5 +; X32-SSE-NEXT: pcmpgtb %xmm4, %xmm5 +; X32-SSE-NEXT: movdqa %xmm5, %xmm6 +; X32-SSE-NEXT: pandn %xmm0, %xmm6 +; X32-SSE-NEXT: psrlw $2, %xmm0 +; X32-SSE-NEXT: pand .LCPI3_4, %xmm0 +; X32-SSE-NEXT: pand %xmm5, %xmm0 +; X32-SSE-NEXT: por %xmm6, %xmm0 +; X32-SSE-NEXT: paddb %xmm4, %xmm4 +; X32-SSE-NEXT: pcmpgtb %xmm4, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pandn %xmm0, %xmm4 +; X32-SSE-NEXT: psrlw $1, %xmm0 +; X32-SSE-NEXT: pand .LCPI3_5, %xmm0 +; X32-SSE-NEXT: pand %xmm3, %xmm0 +; X32-SSE-NEXT: por %xmm4, %xmm0 +; X32-SSE-NEXT: por %xmm1, %xmm0 +; X32-SSE-NEXT: por %xmm2, %xmm0 +; X32-SSE-NEXT: retl + %b8 = sub <16 x i8> , %b + %shl = shl <16 x i8> %a, %b + %lshr = lshr <16 x i8> %a, %b8 + %or = or <16 x i8> %shl, %lshr + ret <16 x i8> %or +} + +; +; Constant Rotates +; + +define <2 x i64> @constant_rotate_v2i64(<2 x i64> %a) nounwind { +; SSE2-LABEL: constant_rotate_v2i64: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: psllq $14, %xmm2 +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: psllq $4, %xmm1 +; SSE2-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1] +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: psrlq $50, %xmm1 +; SSE2-NEXT: psrlq $60, %xmm0 +; SSE2-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1] +; SSE2-NEXT: orpd %xmm2, %xmm1 +; SSE2-NEXT: movapd %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: constant_rotate_v2i64: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: psllq $14, %xmm1 +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: psllq $4, %xmm2 +; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5,6,7] +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: psrlq $50, %xmm1 +; SSE41-NEXT: psrlq $60, %xmm0 +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7] +; SSE41-NEXT: por %xmm2, %xmm0 +; SSE41-NEXT: retq +; +; AVX1-LABEL: constant_rotate_v2i64: +; AVX1: # BB#0: +; AVX1-NEXT: vpsllq $14, %xmm0, %xmm1 +; AVX1-NEXT: vpsllq $4, %xmm0, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7] +; AVX1-NEXT: vpsrlq $50, %xmm0, %xmm2 +; AVX1-NEXT: vpsrlq $60, %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: constant_rotate_v2i64: +; AVX2: # BB#0: +; AVX2-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm1 +; AVX2-NEXT: vpsrlvq {{.*}}(%rip), %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_rotate_v2i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpshlq {{.*}}(%rip), %xmm0, %xmm1 +; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpsubq {{.*}}(%rip), %xmm2, %xmm2 +; XOPAVX1-NEXT: vpshlq %xmm2, %xmm0, %xmm0 +; XOPAVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_rotate_v2i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm1 +; XOPAVX2-NEXT: vpsrlvq {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 +; XOPAVX2-NEXT: retq +; +; X32-SSE-LABEL: constant_rotate_v2i64: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm0, %xmm2 +; X32-SSE-NEXT: psllq $14, %xmm2 +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psllq $4, %xmm1 +; X32-SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1] +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psrlq $50, %xmm1 +; X32-SSE-NEXT: psrlq $60, %xmm0 +; X32-SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1] +; X32-SSE-NEXT: orpd %xmm2, %xmm1 +; X32-SSE-NEXT: movapd %xmm1, %xmm0 +; X32-SSE-NEXT: retl + %shl = shl <2 x i64> %a, + %lshr = lshr <2 x i64> %a, + %or = or <2 x i64> %shl, %lshr + ret <2 x i64> %or +} + +define <4 x i32> @constant_rotate_v4i32(<4 x i32> %a) nounwind { +; SSE2-LABEL: constant_rotate_v4i32: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [16,32,64,128] +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pmuludq %xmm1, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] +; SSE2-NEXT: pmuludq %xmm1, %xmm3 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3] +; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: psrld $25, %xmm1 +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: psrld $27, %xmm3 +; SSE2-NEXT: movsd {{.*#+}} xmm1 = xmm3[0],xmm1[1] +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: psrld $26, %xmm3 +; SSE2-NEXT: psrld $28, %xmm0 +; SSE2-NEXT: movsd {{.*#+}} xmm3 = xmm0[0],xmm3[1] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3] +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE2-NEXT: por %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: constant_rotate_v4i32: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [16,32,64,128] +; SSE41-NEXT: pmulld %xmm0, %xmm1 +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: psrld $25, %xmm2 +; SSE41-NEXT: movdqa %xmm0, %xmm3 +; SSE41-NEXT: psrld $27, %xmm3 +; SSE41-NEXT: pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7] +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: psrld $26, %xmm2 +; SSE41-NEXT: psrld $28, %xmm0 +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7] +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7] +; SSE41-NEXT: por %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; AVX1-LABEL: constant_rotate_v4i32: +; AVX1: # BB#0: +; AVX1-NEXT: vpmulld {{.*}}(%rip), %xmm0, %xmm1 +; AVX1-NEXT: vpsrld $25, %xmm0, %xmm2 +; AVX1-NEXT: vpsrld $27, %xmm0, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vpsrld $26, %xmm0, %xmm3 +; AVX1-NEXT: vpsrld $28, %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7] +; AVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: constant_rotate_v4i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm1 +; AVX2-NEXT: vpsrlvd {{.*}}(%rip), %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_rotate_v4i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm1 +; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_rotate_v4i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm1 +; XOPAVX2-NEXT: vpsrlvd {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 +; XOPAVX2-NEXT: retq +; +; X32-SSE-LABEL: constant_rotate_v4i32: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [16,32,64,128] +; X32-SSE-NEXT: movdqa %xmm0, %xmm2 +; X32-SSE-NEXT: pmuludq %xmm1, %xmm2 +; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3] +; X32-SSE-NEXT: pmuludq %xmm1, %xmm3 +; X32-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3] +; X32-SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psrld $25, %xmm1 +; X32-SSE-NEXT: movdqa %xmm0, %xmm3 +; X32-SSE-NEXT: psrld $27, %xmm3 +; X32-SSE-NEXT: movsd {{.*#+}} xmm1 = xmm3[0],xmm1[1] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3] +; X32-SSE-NEXT: movdqa %xmm0, %xmm3 +; X32-SSE-NEXT: psrld $26, %xmm3 +; X32-SSE-NEXT: psrld $28, %xmm0 +; X32-SSE-NEXT: movsd {{.*#+}} xmm3 = xmm0[0],xmm3[1] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3] +; X32-SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; X32-SSE-NEXT: por %xmm2, %xmm0 +; X32-SSE-NEXT: retl + %shl = shl <4 x i32> %a, + %lshr = lshr <4 x i32> %a, + %or = or <4 x i32> %shl, %lshr + ret <4 x i32> %or +} + +define <8 x i16> @constant_rotate_v8i16(<8 x i16> %a) nounwind { +; SSE2-LABEL: constant_rotate_v8i16: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128] +; SSE2-NEXT: pmullw %xmm0, %xmm2 +; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: psrlw $8, %xmm0 +; SSE2-NEXT: pand %xmm1, %xmm0 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,0,0,0] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: psrlw $4, %xmm0 +; SSE2-NEXT: pand %xmm1, %xmm0 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [0,65535,65535,0,0,65535,65535,0] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: psrlw $2, %xmm0 +; SSE2-NEXT: pand %xmm1, %xmm0 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [65535,0,65535,0,65535,0,65535,0] +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: pand %xmm3, %xmm1 +; SSE2-NEXT: psrlw $1, %xmm0 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: constant_rotate_v8i16: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128] +; SSE41-NEXT: pmullw %xmm1, %xmm2 +; SSE41-NEXT: movdqa %xmm1, %xmm3 +; SSE41-NEXT: psrlw $8, %xmm3 +; SSE41-NEXT: movaps {{.*#+}} xmm0 = [256,61680,57568,53456,49344,45232,41120,37008] +; SSE41-NEXT: pblendvb %xmm3, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm3 +; SSE41-NEXT: psrlw $4, %xmm3 +; SSE41-NEXT: movaps {{.*#+}} xmm0 = [512,57824,49600,41376,33152,24928,16704,8480] +; SSE41-NEXT: pblendvb %xmm3, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm3 +; SSE41-NEXT: psrlw $2, %xmm3 +; SSE41-NEXT: movaps {{.*#+}} xmm0 = [1024,50112,33664,17216,768,49856,33408,16960] +; SSE41-NEXT: pblendvb %xmm3, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm3 +; SSE41-NEXT: psrlw $1, %xmm3 +; SSE41-NEXT: movaps {{.*#+}} xmm0 = [2048,34688,1792,34432,1536,34176,1280,33920] +; SSE41-NEXT: pblendvb %xmm3, %xmm1 +; SSE41-NEXT: por %xmm2, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; AVX1-LABEL: constant_rotate_v8i16: +; AVX1: # BB#0: +; AVX1-NEXT: vpmullw {{.*}}(%rip), %xmm0, %xmm1 +; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [256,61680,57568,53456,49344,45232,41120,37008] +; AVX1-NEXT: vpblendvb %xmm3, %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [512,57824,49600,41376,33152,24928,16704,8480] +; AVX1-NEXT: vpblendvb %xmm3, %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $2, %xmm0, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [1024,50112,33664,17216,768,49856,33408,16960] +; AVX1-NEXT: vpblendvb %xmm3, %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $1, %xmm0, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [2048,34688,1792,34432,1536,34176,1280,33920] +; AVX1-NEXT: vpblendvb %xmm3, %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: constant_rotate_v8i16: +; AVX2: # BB#0: +; AVX2-NEXT: vpmullw {{.*}}(%rip), %xmm0, %xmm1 +; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero +; AVX2-NEXT: vpsrlvd %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,20,21,24,25,28,29],zero,zero,zero,zero,zero,zero,zero,zero +; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] +; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; XOP-LABEL: constant_rotate_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpshlw {{.*}}(%rip), %xmm0, %xmm1 +; XOP-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOP-NEXT: vpsubw {{.*}}(%rip), %xmm2, %xmm2 +; XOP-NEXT: vpshlw %xmm2, %xmm0, %xmm0 +; XOP-NEXT: vpor %xmm0, %xmm1, %xmm0 +; XOP-NEXT: retq +; +; X32-SSE-LABEL: constant_rotate_v8i16: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128] +; X32-SSE-NEXT: pmullw %xmm0, %xmm2 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535] +; X32-SSE-NEXT: movdqa %xmm1, %xmm3 +; X32-SSE-NEXT: pandn %xmm0, %xmm3 +; X32-SSE-NEXT: psrlw $8, %xmm0 +; X32-SSE-NEXT: pand %xmm1, %xmm0 +; X32-SSE-NEXT: por %xmm3, %xmm0 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,0,0,0] +; X32-SSE-NEXT: movdqa %xmm1, %xmm3 +; X32-SSE-NEXT: pandn %xmm0, %xmm3 +; X32-SSE-NEXT: psrlw $4, %xmm0 +; X32-SSE-NEXT: pand %xmm1, %xmm0 +; X32-SSE-NEXT: por %xmm3, %xmm0 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [0,65535,65535,0,0,65535,65535,0] +; X32-SSE-NEXT: movdqa %xmm1, %xmm3 +; X32-SSE-NEXT: pandn %xmm0, %xmm3 +; X32-SSE-NEXT: psrlw $2, %xmm0 +; X32-SSE-NEXT: pand %xmm1, %xmm0 +; X32-SSE-NEXT: por %xmm3, %xmm0 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [65535,0,65535,0,65535,0,65535,0] +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: pand %xmm3, %xmm1 +; X32-SSE-NEXT: psrlw $1, %xmm0 +; X32-SSE-NEXT: pandn %xmm0, %xmm3 +; X32-SSE-NEXT: por %xmm2, %xmm3 +; X32-SSE-NEXT: por %xmm3, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm0 +; X32-SSE-NEXT: retl + %shl = shl <8 x i16> %a, + %lshr = lshr <8 x i16> %a, + %or = or <8 x i16> %shl, %lshr + ret <8 x i16> %or +} + +define <16 x i8> @constant_rotate_v16i8(<16 x i8> %a) nounwind { +; SSE2-LABEL: constant_rotate_v16i8: +; SSE2: # BB#0: +; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,6,7,8,7,6,5,4,3,2,1] +; SSE2-NEXT: psllw $5, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: pcmpgtb %xmm3, %xmm1 +; SSE2-NEXT: movdqa %xmm0, %xmm4 +; SSE2-NEXT: psllw $4, %xmm4 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm4 +; SSE2-NEXT: pand %xmm1, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm1 +; SSE2-NEXT: por %xmm4, %xmm1 +; SSE2-NEXT: paddb %xmm3, %xmm3 +; SSE2-NEXT: pxor %xmm4, %xmm4 +; SSE2-NEXT: pcmpgtb %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm4, %xmm5 +; SSE2-NEXT: pandn %xmm1, %xmm5 +; SSE2-NEXT: psllw $2, %xmm1 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm1 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: por %xmm5, %xmm1 +; SSE2-NEXT: paddb %xmm3, %xmm3 +; SSE2-NEXT: pxor %xmm4, %xmm4 +; SSE2-NEXT: pcmpgtb %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm4, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: paddb %xmm1, %xmm1 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [8,7,6,5,4,3,2,1,0,1,2,3,4,5,6,7] +; SSE2-NEXT: psllw $5, %xmm4 +; SSE2-NEXT: pxor %xmm5, %xmm5 +; SSE2-NEXT: pcmpgtb %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm6 +; SSE2-NEXT: pandn %xmm0, %xmm6 +; SSE2-NEXT: psrlw $4, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: por %xmm6, %xmm0 +; SSE2-NEXT: paddb %xmm4, %xmm4 +; SSE2-NEXT: pxor %xmm5, %xmm5 +; SSE2-NEXT: pcmpgtb %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm6 +; SSE2-NEXT: pandn %xmm0, %xmm6 +; SSE2-NEXT: psrlw $2, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: por %xmm6, %xmm0 +; SSE2-NEXT: paddb %xmm4, %xmm4 +; SSE2-NEXT: pcmpgtb %xmm4, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm4 +; SSE2-NEXT: psrlw $1, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: por %xmm4, %xmm0 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: por %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: constant_rotate_v16i8: +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,8,7,6,5,4,3,2,1] +; SSE41-NEXT: psllw $5, %xmm0 +; SSE41-NEXT: movdqa %xmm1, %xmm3 +; SSE41-NEXT: psllw $4, %xmm3 +; SSE41-NEXT: pand {{.*}}(%rip), %xmm3 +; SSE41-NEXT: movdqa %xmm1, %xmm2 +; SSE41-NEXT: pblendvb %xmm3, %xmm2 +; SSE41-NEXT: movdqa %xmm2, %xmm3 +; SSE41-NEXT: psllw $2, %xmm3 +; SSE41-NEXT: pand {{.*}}(%rip), %xmm3 +; SSE41-NEXT: paddb %xmm0, %xmm0 +; SSE41-NEXT: pblendvb %xmm3, %xmm2 +; SSE41-NEXT: movdqa %xmm2, %xmm3 +; SSE41-NEXT: paddb %xmm3, %xmm3 +; SSE41-NEXT: paddb %xmm0, %xmm0 +; SSE41-NEXT: pblendvb %xmm3, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [8,7,6,5,4,3,2,1,0,1,2,3,4,5,6,7] +; SSE41-NEXT: psllw $5, %xmm0 +; SSE41-NEXT: movdqa %xmm1, %xmm3 +; SSE41-NEXT: psrlw $4, %xmm3 +; SSE41-NEXT: pand {{.*}}(%rip), %xmm3 +; SSE41-NEXT: pblendvb %xmm3, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm3 +; SSE41-NEXT: psrlw $2, %xmm3 +; SSE41-NEXT: pand {{.*}}(%rip), %xmm3 +; SSE41-NEXT: paddb %xmm0, %xmm0 +; SSE41-NEXT: pblendvb %xmm3, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm3 +; SSE41-NEXT: psrlw $1, %xmm3 +; SSE41-NEXT: pand {{.*}}(%rip), %xmm3 +; SSE41-NEXT: paddb %xmm0, %xmm0 +; SSE41-NEXT: pblendvb %xmm3, %xmm1 +; SSE41-NEXT: por %xmm2, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: constant_rotate_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,8,7,6,5,4,3,2,1] +; AVX-NEXT: vpsllw $5, %xmm1, %xmm1 +; AVX-NEXT: vpsllw $4, %xmm0, %xmm2 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm2 +; AVX-NEXT: vpsllw $2, %xmm2, %xmm3 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm3, %xmm3 +; AVX-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpblendvb %xmm1, %xmm3, %xmm2, %xmm2 +; AVX-NEXT: vpaddb %xmm2, %xmm2, %xmm3 +; AVX-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpblendvb %xmm1, %xmm3, %xmm2, %xmm1 +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [8,7,6,5,4,3,2,1,0,1,2,3,4,5,6,7] +; AVX-NEXT: vpsllw $5, %xmm2, %xmm2 +; AVX-NEXT: vpsrlw $4, %xmm0, %xmm3 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm3, %xmm3 +; AVX-NEXT: vpblendvb %xmm2, %xmm3, %xmm0, %xmm0 +; AVX-NEXT: vpsrlw $2, %xmm0, %xmm3 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm3, %xmm3 +; AVX-NEXT: vpaddb %xmm2, %xmm2, %xmm2 +; AVX-NEXT: vpblendvb %xmm2, %xmm3, %xmm0, %xmm0 +; AVX-NEXT: vpsrlw $1, %xmm0, %xmm3 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm3, %xmm3 +; AVX-NEXT: vpaddb %xmm2, %xmm2, %xmm2 +; AVX-NEXT: vpblendvb %xmm2, %xmm3, %xmm0, %xmm0 +; AVX-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: constant_rotate_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpshlb {{.*}}(%rip), %xmm0, %xmm1 +; XOP-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOP-NEXT: vpsubb {{.*}}(%rip), %xmm2, %xmm2 +; XOP-NEXT: vpshlb %xmm2, %xmm0, %xmm0 +; XOP-NEXT: vpor %xmm0, %xmm1, %xmm0 +; XOP-NEXT: retq +; +; X32-SSE-LABEL: constant_rotate_v16i8: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,6,7,8,7,6,5,4,3,2,1] +; X32-SSE-NEXT: psllw $5, %xmm3 +; X32-SSE-NEXT: pxor %xmm2, %xmm2 +; X32-SSE-NEXT: pxor %xmm1, %xmm1 +; X32-SSE-NEXT: pcmpgtb %xmm3, %xmm1 +; X32-SSE-NEXT: movdqa %xmm0, %xmm4 +; X32-SSE-NEXT: psllw $4, %xmm4 +; X32-SSE-NEXT: pand .LCPI7_1, %xmm4 +; X32-SSE-NEXT: pand %xmm1, %xmm4 +; X32-SSE-NEXT: pandn %xmm0, %xmm1 +; X32-SSE-NEXT: por %xmm4, %xmm1 +; X32-SSE-NEXT: paddb %xmm3, %xmm3 +; X32-SSE-NEXT: pxor %xmm4, %xmm4 +; X32-SSE-NEXT: pcmpgtb %xmm3, %xmm4 +; X32-SSE-NEXT: movdqa %xmm4, %xmm5 +; X32-SSE-NEXT: pandn %xmm1, %xmm5 +; X32-SSE-NEXT: psllw $2, %xmm1 +; X32-SSE-NEXT: pand .LCPI7_2, %xmm1 +; X32-SSE-NEXT: pand %xmm4, %xmm1 +; X32-SSE-NEXT: por %xmm5, %xmm1 +; X32-SSE-NEXT: paddb %xmm3, %xmm3 +; X32-SSE-NEXT: pxor %xmm4, %xmm4 +; X32-SSE-NEXT: pcmpgtb %xmm3, %xmm4 +; X32-SSE-NEXT: movdqa %xmm4, %xmm3 +; X32-SSE-NEXT: pandn %xmm1, %xmm3 +; X32-SSE-NEXT: paddb %xmm1, %xmm1 +; X32-SSE-NEXT: pand %xmm4, %xmm1 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm4 = [8,7,6,5,4,3,2,1,0,1,2,3,4,5,6,7] +; X32-SSE-NEXT: psllw $5, %xmm4 +; X32-SSE-NEXT: pxor %xmm5, %xmm5 +; X32-SSE-NEXT: pcmpgtb %xmm4, %xmm5 +; X32-SSE-NEXT: movdqa %xmm5, %xmm6 +; X32-SSE-NEXT: pandn %xmm0, %xmm6 +; X32-SSE-NEXT: psrlw $4, %xmm0 +; X32-SSE-NEXT: pand .LCPI7_4, %xmm0 +; X32-SSE-NEXT: pand %xmm5, %xmm0 +; X32-SSE-NEXT: por %xmm6, %xmm0 +; X32-SSE-NEXT: paddb %xmm4, %xmm4 +; X32-SSE-NEXT: pxor %xmm5, %xmm5 +; X32-SSE-NEXT: pcmpgtb %xmm4, %xmm5 +; X32-SSE-NEXT: movdqa %xmm5, %xmm6 +; X32-SSE-NEXT: pandn %xmm0, %xmm6 +; X32-SSE-NEXT: psrlw $2, %xmm0 +; X32-SSE-NEXT: pand .LCPI7_5, %xmm0 +; X32-SSE-NEXT: pand %xmm5, %xmm0 +; X32-SSE-NEXT: por %xmm6, %xmm0 +; X32-SSE-NEXT: paddb %xmm4, %xmm4 +; X32-SSE-NEXT: pcmpgtb %xmm4, %xmm2 +; X32-SSE-NEXT: movdqa %xmm2, %xmm4 +; X32-SSE-NEXT: pandn %xmm0, %xmm4 +; X32-SSE-NEXT: psrlw $1, %xmm0 +; X32-SSE-NEXT: pand .LCPI7_6, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: por %xmm4, %xmm0 +; X32-SSE-NEXT: por %xmm3, %xmm0 +; X32-SSE-NEXT: por %xmm1, %xmm0 +; X32-SSE-NEXT: retl + %shl = shl <16 x i8> %a, + %lshr = lshr <16 x i8> %a, + %or = or <16 x i8> %shl, %lshr + ret <16 x i8> %or +} + +; +; Uniform Constant Rotates +; + +define <2 x i64> @splatconstant_rotate_v2i64(<2 x i64> %a) nounwind { +; SSE-LABEL: splatconstant_rotate_v2i64: +; SSE: # BB#0: +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psllq $14, %xmm1 +; SSE-NEXT: psrlq $50, %xmm0 +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: splatconstant_rotate_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpsllq $14, %xmm0, %xmm1 +; AVX-NEXT: vpsrlq $50, %xmm0, %xmm0 +; AVX-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_rotate_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vprotq $14, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_rotate_v2i64: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psllq $14, %xmm1 +; X32-SSE-NEXT: psrlq $50, %xmm0 +; X32-SSE-NEXT: por %xmm1, %xmm0 +; X32-SSE-NEXT: retl + %shl = shl <2 x i64> %a, + %lshr = lshr <2 x i64> %a, + %or = or <2 x i64> %shl, %lshr + ret <2 x i64> %or +} + +define <4 x i32> @splatconstant_rotate_v4i32(<4 x i32> %a) nounwind { +; SSE-LABEL: splatconstant_rotate_v4i32: +; SSE: # BB#0: +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: pslld $4, %xmm1 +; SSE-NEXT: psrld $28, %xmm0 +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: splatconstant_rotate_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpslld $4, %xmm0, %xmm1 +; AVX-NEXT: vpsrld $28, %xmm0, %xmm0 +; AVX-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_rotate_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vprotd $4, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_rotate_v4i32: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: pslld $4, %xmm1 +; X32-SSE-NEXT: psrld $28, %xmm0 +; X32-SSE-NEXT: por %xmm1, %xmm0 +; X32-SSE-NEXT: retl + %shl = shl <4 x i32> %a, + %lshr = lshr <4 x i32> %a, + %or = or <4 x i32> %shl, %lshr + ret <4 x i32> %or +} + +define <8 x i16> @splatconstant_rotate_v8i16(<8 x i16> %a) nounwind { +; SSE-LABEL: splatconstant_rotate_v8i16: +; SSE: # BB#0: +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psllw $7, %xmm1 +; SSE-NEXT: psrlw $9, %xmm0 +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: splatconstant_rotate_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpsllw $7, %xmm0, %xmm1 +; AVX-NEXT: vpsrlw $9, %xmm0, %xmm0 +; AVX-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_rotate_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vprotw $7, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_rotate_v8i16: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psllw $7, %xmm1 +; X32-SSE-NEXT: psrlw $9, %xmm0 +; X32-SSE-NEXT: por %xmm1, %xmm0 +; X32-SSE-NEXT: retl + %shl = shl <8 x i16> %a, + %lshr = lshr <8 x i16> %a, + %or = or <8 x i16> %shl, %lshr + ret <8 x i16> %or +} + +define <16 x i8> @splatconstant_rotate_v16i8(<16 x i8> %a) nounwind { +; SSE-LABEL: splatconstant_rotate_v16i8: +; SSE: # BB#0: +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psllw $4, %xmm1 +; SSE-NEXT: pand {{.*}}(%rip), %xmm1 +; SSE-NEXT: psrlw $4, %xmm0 +; SSE-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: splatconstant_rotate_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpsllw $4, %xmm0, %xmm1 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm1, %xmm1 +; AVX-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_rotate_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vprotb $4, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_rotate_v16i8: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psllw $4, %xmm1 +; X32-SSE-NEXT: pand .LCPI11_0, %xmm1 +; X32-SSE-NEXT: psrlw $4, %xmm0 +; X32-SSE-NEXT: pand .LCPI11_1, %xmm0 +; X32-SSE-NEXT: por %xmm1, %xmm0 +; X32-SSE-NEXT: retl + %shl = shl <16 x i8> %a, + %lshr = lshr <16 x i8> %a, + %or = or <16 x i8> %shl, %lshr + ret <16 x i8> %or +} + +; +; Masked Uniform Constant Rotates +; + +define <2 x i64> @splatconstant_rotate_mask_v2i64(<2 x i64> %a) nounwind { +; SSE-LABEL: splatconstant_rotate_mask_v2i64: +; SSE: # BB#0: +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psllq $15, %xmm1 +; SSE-NEXT: psrlq $49, %xmm0 +; SSE-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE-NEXT: pand {{.*}}(%rip), %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: splatconstant_rotate_mask_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpsllq $15, %xmm0, %xmm1 +; AVX-NEXT: vpsrlq $49, %xmm0, %xmm0 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm1, %xmm1 +; AVX-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_rotate_mask_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vprotq $15, %xmm0, %xmm0 +; XOP-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; XOP-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_rotate_mask_v2i64: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psllq $15, %xmm1 +; X32-SSE-NEXT: psrlq $49, %xmm0 +; X32-SSE-NEXT: pand .LCPI12_0, %xmm0 +; X32-SSE-NEXT: pand .LCPI12_1, %xmm1 +; X32-SSE-NEXT: por %xmm0, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm0 +; X32-SSE-NEXT: retl + %shl = shl <2 x i64> %a, + %lshr = lshr <2 x i64> %a, + %rmask = and <2 x i64> %lshr, + %lmask = and <2 x i64> %shl, + %or = or <2 x i64> %lmask, %rmask + ret <2 x i64> %or +} + +define <4 x i32> @splatconstant_rotate_mask_v4i32(<4 x i32> %a) nounwind { +; SSE-LABEL: splatconstant_rotate_mask_v4i32: +; SSE: # BB#0: +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: pslld $4, %xmm1 +; SSE-NEXT: psrld $28, %xmm0 +; SSE-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE-NEXT: pand {{.*}}(%rip), %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: splatconstant_rotate_mask_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpslld $4, %xmm0, %xmm1 +; AVX-NEXT: vpsrld $28, %xmm0, %xmm0 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm1, %xmm1 +; AVX-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_rotate_mask_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vprotd $4, %xmm0, %xmm0 +; XOP-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; XOP-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_rotate_mask_v4i32: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: pslld $4, %xmm1 +; X32-SSE-NEXT: psrld $28, %xmm0 +; X32-SSE-NEXT: pand .LCPI13_0, %xmm0 +; X32-SSE-NEXT: pand .LCPI13_1, %xmm1 +; X32-SSE-NEXT: por %xmm0, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm0 +; X32-SSE-NEXT: retl + %shl = shl <4 x i32> %a, + %lshr = lshr <4 x i32> %a, + %rmask = and <4 x i32> %lshr, + %lmask = and <4 x i32> %shl, + %or = or <4 x i32> %lmask, %rmask + ret <4 x i32> %or +} + +define <8 x i16> @splatconstant_rotate_mask_v8i16(<8 x i16> %a) nounwind { +; SSE-LABEL: splatconstant_rotate_mask_v8i16: +; SSE: # BB#0: +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psllw $5, %xmm1 +; SSE-NEXT: psrlw $11, %xmm0 +; SSE-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE-NEXT: pand {{.*}}(%rip), %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: splatconstant_rotate_mask_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpsllw $5, %xmm0, %xmm1 +; AVX-NEXT: vpsrlw $11, %xmm0, %xmm0 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm1, %xmm1 +; AVX-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_rotate_mask_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vprotw $5, %xmm0, %xmm0 +; XOP-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; XOP-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_rotate_mask_v8i16: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psllw $5, %xmm1 +; X32-SSE-NEXT: psrlw $11, %xmm0 +; X32-SSE-NEXT: pand .LCPI14_0, %xmm0 +; X32-SSE-NEXT: pand .LCPI14_1, %xmm1 +; X32-SSE-NEXT: por %xmm0, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm0 +; X32-SSE-NEXT: retl + %shl = shl <8 x i16> %a, + %lshr = lshr <8 x i16> %a, + %rmask = and <8 x i16> %lshr, + %lmask = and <8 x i16> %shl, + %or = or <8 x i16> %lmask, %rmask + ret <8 x i16> %or +} + +define <16 x i8> @splatconstant_rotate_mask_v16i8(<16 x i8> %a) nounwind { +; SSE-LABEL: splatconstant_rotate_mask_v16i8: +; SSE: # BB#0: +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: psllw $4, %xmm1 +; SSE-NEXT: pand {{.*}}(%rip), %xmm1 +; SSE-NEXT: psrlw $4, %xmm0 +; SSE-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE-NEXT: pand {{.*}}(%rip), %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: splatconstant_rotate_mask_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpsllw $4, %xmm0, %xmm1 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm1, %xmm1 +; AVX-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm1, %xmm1 +; AVX-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_rotate_mask_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vprotb $4, %xmm0, %xmm0 +; XOP-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; XOP-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_rotate_mask_v16i8: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psllw $4, %xmm1 +; X32-SSE-NEXT: pand .LCPI15_0, %xmm1 +; X32-SSE-NEXT: psrlw $4, %xmm0 +; X32-SSE-NEXT: pand .LCPI15_1, %xmm0 +; X32-SSE-NEXT: pand .LCPI15_2, %xmm0 +; X32-SSE-NEXT: pand .LCPI15_3, %xmm1 +; X32-SSE-NEXT: por %xmm0, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm0 +; X32-SSE-NEXT: retl + %shl = shl <16 x i8> %a, + %lshr = lshr <16 x i8> %a, + %rmask = and <16 x i8> %lshr, + %lmask = and <16 x i8> %shl, + %or = or <16 x i8> %lmask, %rmask + ret <16 x i8> %or +} diff --git a/test/CodeGen/X86/vector-rotate-256.ll b/test/CodeGen/X86/vector-rotate-256.ll new file mode 100644 index 000000000000..379b5fcb635f --- /dev/null +++ b/test/CodeGen/X86/vector-rotate-256.ll @@ -0,0 +1,1089 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX2 + +; +; Variable Rotates +; + +define <4 x i64> @var_rotate_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind { +; AVX1-LABEL: var_rotate_v4i64: +; AVX1: # BB#0: +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [64,64] +; AVX1-NEXT: vpsubq %xmm1, %xmm2, %xmm3 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpsubq %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpsllq %xmm4, %xmm5, %xmm6 +; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,3,0,1] +; AVX1-NEXT: vpsllq %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1,2,3],xmm4[4,5,6,7] +; AVX1-NEXT: vpsllq %xmm1, %xmm0, %xmm6 +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1] +; AVX1-NEXT: vpsllq %xmm1, %xmm0, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm6[0,1,2,3],xmm1[4,5,6,7] +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm1, %ymm1 +; AVX1-NEXT: vpsrlq %xmm2, %xmm5, %xmm4 +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1] +; AVX1-NEXT: vpsrlq %xmm2, %xmm5, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vpsrlq %xmm3, %xmm0, %xmm4 +; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1] +; AVX1-NEXT: vpsrlq %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm4[0,1,2,3],xmm0[4,5,6,7] +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: var_rotate_v4i64: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpsubq %ymm1, %ymm2, %ymm2 +; AVX2-NEXT: vpsllvq %ymm1, %ymm0, %ymm1 +; AVX2-NEXT: vpsrlvq %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_rotate_v4i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; XOPAVX1-NEXT: vprotq %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vprotq %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_rotate_v4i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm3 +; XOPAVX2-NEXT: vprotq %xmm2, %xmm3, %xmm2 +; XOPAVX2-NEXT: vprotq %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq + %b64 = sub <4 x i64> , %b + %shl = shl <4 x i64> %a, %b + %lshr = lshr <4 x i64> %a, %b64 + %or = or <4 x i64> %shl, %lshr + ret <4 x i64> %or +} + +define <8 x i32> @var_rotate_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind { +; AVX1-LABEL: var_rotate_v8i32: +; AVX1: # BB#0: +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [32,32,32,32] +; AVX1-NEXT: vpsubd %xmm1, %xmm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpsubd %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vpslld $23, %xmm4, %xmm4 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216] +; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4 +; AVX1-NEXT: vcvttps2dq %xmm4, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6 +; AVX1-NEXT: vpmulld %xmm6, %xmm4, %xmm4 +; AVX1-NEXT: vpslld $23, %xmm1, %xmm1 +; AVX1-NEXT: vpaddd %xmm5, %xmm1, %xmm1 +; AVX1-NEXT: vcvttps2dq %xmm1, %xmm1 +; AVX1-NEXT: vpmulld %xmm0, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm1, %ymm1 +; AVX1-NEXT: vpsrldq {{.*#+}} xmm4 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpsrld %xmm4, %xmm6, %xmm4 +; AVX1-NEXT: vpsrlq $32, %xmm3, %xmm5 +; AVX1-NEXT: vpsrld %xmm5, %xmm6, %xmm5 +; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7] +; AVX1-NEXT: vpxor %xmm5, %xmm5, %xmm5 +; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm3[2],xmm5[2],xmm3[3],xmm5[3] +; AVX1-NEXT: vpsrld %xmm7, %xmm6, %xmm7 +; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero +; AVX1-NEXT: vpsrld %xmm3, %xmm6, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm7[4,5,6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3],xmm3[4,5],xmm4[6,7] +; AVX1-NEXT: vpsrldq {{.*#+}} xmm4 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpsrld %xmm4, %xmm0, %xmm4 +; AVX1-NEXT: vpsrlq $32, %xmm2, %xmm6 +; AVX1-NEXT: vpsrld %xmm6, %xmm0, %xmm6 +; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1,2,3],xmm4[4,5,6,7] +; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm2[2],xmm5[2],xmm2[3],xmm5[3] +; AVX1-NEXT: vpsrld %xmm5, %xmm0, %xmm5 +; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero +; AVX1-NEXT: vpsrld %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm5[4,5,6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm4[2,3],xmm0[4,5],xmm4[6,7] +; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0 +; AVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: var_rotate_v8i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpsubd %ymm1, %ymm2, %ymm2 +; AVX2-NEXT: vpsllvd %ymm1, %ymm0, %ymm1 +; AVX2-NEXT: vpsrlvd %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_rotate_v8i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; XOPAVX1-NEXT: vprotd %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vprotd %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_rotate_v8i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm3 +; XOPAVX2-NEXT: vprotd %xmm2, %xmm3, %xmm2 +; XOPAVX2-NEXT: vprotd %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq + %b32 = sub <8 x i32> , %b + %shl = shl <8 x i32> %a, %b + %lshr = lshr <8 x i32> %a, %b32 + %or = or <8 x i32> %shl, %lshr + ret <8 x i32> %or +} + +define <16 x i16> @var_rotate_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind { +; AVX1-LABEL: var_rotate_v16i16: +; AVX1: # BB#0: +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16] +; AVX1-NEXT: vpsubw %xmm1, %xmm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpsubw %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vpsllw $12, %xmm4, %xmm5 +; AVX1-NEXT: vpsllw $4, %xmm4, %xmm4 +; AVX1-NEXT: vpor %xmm5, %xmm4, %xmm5 +; AVX1-NEXT: vpaddw %xmm5, %xmm5, %xmm6 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; AVX1-NEXT: vpsllw $8, %xmm4, %xmm7 +; AVX1-NEXT: vpblendvb %xmm5, %xmm7, %xmm4, %xmm5 +; AVX1-NEXT: vpsllw $4, %xmm5, %xmm7 +; AVX1-NEXT: vpblendvb %xmm6, %xmm7, %xmm5, %xmm5 +; AVX1-NEXT: vpsllw $2, %xmm5, %xmm7 +; AVX1-NEXT: vpaddw %xmm6, %xmm6, %xmm6 +; AVX1-NEXT: vpblendvb %xmm6, %xmm7, %xmm5, %xmm5 +; AVX1-NEXT: vpsllw $1, %xmm5, %xmm7 +; AVX1-NEXT: vpaddw %xmm6, %xmm6, %xmm6 +; AVX1-NEXT: vpblendvb %xmm6, %xmm7, %xmm5, %xmm5 +; AVX1-NEXT: vpsllw $12, %xmm1, %xmm6 +; AVX1-NEXT: vpsllw $4, %xmm1, %xmm1 +; AVX1-NEXT: vpor %xmm6, %xmm1, %xmm1 +; AVX1-NEXT: vpaddw %xmm1, %xmm1, %xmm6 +; AVX1-NEXT: vpsllw $8, %xmm0, %xmm7 +; AVX1-NEXT: vpblendvb %xmm1, %xmm7, %xmm0, %xmm1 +; AVX1-NEXT: vpsllw $4, %xmm1, %xmm7 +; AVX1-NEXT: vpblendvb %xmm6, %xmm7, %xmm1, %xmm1 +; AVX1-NEXT: vpsllw $2, %xmm1, %xmm7 +; AVX1-NEXT: vpaddw %xmm6, %xmm6, %xmm6 +; AVX1-NEXT: vpblendvb %xmm6, %xmm7, %xmm1, %xmm1 +; AVX1-NEXT: vpsllw $1, %xmm1, %xmm7 +; AVX1-NEXT: vpaddw %xmm6, %xmm6, %xmm6 +; AVX1-NEXT: vpblendvb %xmm6, %xmm7, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm1, %ymm1 +; AVX1-NEXT: vpsllw $12, %xmm3, %xmm5 +; AVX1-NEXT: vpsllw $4, %xmm3, %xmm3 +; AVX1-NEXT: vpor %xmm5, %xmm3, %xmm3 +; AVX1-NEXT: vpaddw %xmm3, %xmm3, %xmm5 +; AVX1-NEXT: vpsrlw $8, %xmm4, %xmm6 +; AVX1-NEXT: vpblendvb %xmm3, %xmm6, %xmm4, %xmm3 +; AVX1-NEXT: vpsrlw $4, %xmm3, %xmm4 +; AVX1-NEXT: vpblendvb %xmm5, %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vpsrlw $2, %xmm3, %xmm4 +; AVX1-NEXT: vpaddw %xmm5, %xmm5, %xmm5 +; AVX1-NEXT: vpblendvb %xmm5, %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vpsrlw $1, %xmm3, %xmm4 +; AVX1-NEXT: vpaddw %xmm5, %xmm5, %xmm5 +; AVX1-NEXT: vpblendvb %xmm5, %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vpsllw $12, %xmm2, %xmm4 +; AVX1-NEXT: vpsllw $4, %xmm2, %xmm2 +; AVX1-NEXT: vpor %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vpaddw %xmm2, %xmm2, %xmm4 +; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm5 +; AVX1-NEXT: vpblendvb %xmm2, %xmm5, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm2 +; AVX1-NEXT: vpblendvb %xmm4, %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $2, %xmm0, %xmm2 +; AVX1-NEXT: vpaddw %xmm4, %xmm4, %xmm4 +; AVX1-NEXT: vpblendvb %xmm4, %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $1, %xmm0, %xmm2 +; AVX1-NEXT: vpaddw %xmm4, %xmm4, %xmm4 +; AVX1-NEXT: vpblendvb %xmm4, %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0 +; AVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: var_rotate_v16i16: +; AVX2: # BB#0: +; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] +; AVX2-NEXT: vpsubw %ymm1, %ymm2, %ymm2 +; AVX2-NEXT: vpxor %ymm3, %ymm3, %ymm3 +; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm1[4],ymm3[4],ymm1[5],ymm3[5],ymm1[6],ymm3[6],ymm1[7],ymm3[7],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15] +; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15] +; AVX2-NEXT: vpsllvd %ymm4, %ymm5, %ymm4 +; AVX2-NEXT: vpsrld $16, %ymm4, %ymm4 +; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11] +; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11] +; AVX2-NEXT: vpsllvd %ymm1, %ymm0, %ymm1 +; AVX2-NEXT: vpsrld $16, %ymm1, %ymm1 +; AVX2-NEXT: vpackusdw %ymm4, %ymm1, %ymm1 +; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15] +; AVX2-NEXT: vpsrlvd %ymm4, %ymm5, %ymm4 +; AVX2-NEXT: vpsrld $16, %ymm4, %ymm4 +; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11] +; AVX2-NEXT: vpsrlvd %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0 +; AVX2-NEXT: vpackusdw %ymm4, %ymm0, %ymm0 +; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_rotate_v16i16: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; XOPAVX1-NEXT: vprotw %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vprotw %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_rotate_v16i16: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm3 +; XOPAVX2-NEXT: vprotw %xmm2, %xmm3, %xmm2 +; XOPAVX2-NEXT: vprotw %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq + %b16 = sub <16 x i16> , %b + %shl = shl <16 x i16> %a, %b + %lshr = lshr <16 x i16> %a, %b16 + %or = or <16 x i16> %shl, %lshr + ret <16 x i16> %or +} + +define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind { +; AVX1-LABEL: var_rotate_v32i8: +; AVX1: # BB#0: +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX1-NEXT: vpsubb %xmm1, %xmm3, %xmm8 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpsubb %xmm4, %xmm3, %xmm9 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpsllw $4, %xmm5, %xmm6 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm7 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240] +; AVX1-NEXT: vpand %xmm7, %xmm6, %xmm6 +; AVX1-NEXT: vpsllw $5, %xmm4, %xmm4 +; AVX1-NEXT: vpblendvb %xmm4, %xmm6, %xmm5, %xmm6 +; AVX1-NEXT: vpsllw $2, %xmm6, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252] +; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpaddb %xmm4, %xmm4, %xmm4 +; AVX1-NEXT: vpblendvb %xmm4, %xmm2, %xmm6, %xmm2 +; AVX1-NEXT: vpaddb %xmm2, %xmm2, %xmm6 +; AVX1-NEXT: vpaddb %xmm4, %xmm4, %xmm4 +; AVX1-NEXT: vpblendvb %xmm4, %xmm6, %xmm2, %xmm2 +; AVX1-NEXT: vpsllw $4, %xmm0, %xmm4 +; AVX1-NEXT: vpand %xmm7, %xmm4, %xmm4 +; AVX1-NEXT: vpsllw $5, %xmm1, %xmm1 +; AVX1-NEXT: vpblendvb %xmm1, %xmm4, %xmm0, %xmm4 +; AVX1-NEXT: vpsllw $2, %xmm4, %xmm6 +; AVX1-NEXT: vpand %xmm3, %xmm6, %xmm3 +; AVX1-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendvb %xmm1, %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vpaddb %xmm3, %xmm3, %xmm4 +; AVX1-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendvb %xmm1, %xmm4, %xmm3, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: vpsrlw $4, %xmm5, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpsllw $5, %xmm9, %xmm4 +; AVX1-NEXT: vpblendvb %xmm4, %xmm2, %xmm5, %xmm2 +; AVX1-NEXT: vpsrlw $2, %xmm2, %xmm5 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63] +; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm5 +; AVX1-NEXT: vpaddb %xmm4, %xmm4, %xmm4 +; AVX1-NEXT: vpblendvb %xmm4, %xmm5, %xmm2, %xmm2 +; AVX1-NEXT: vpsrlw $1, %xmm2, %xmm5 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127] +; AVX1-NEXT: vpand %xmm7, %xmm5, %xmm5 +; AVX1-NEXT: vpaddb %xmm4, %xmm4, %xmm4 +; AVX1-NEXT: vpblendvb %xmm4, %xmm5, %xmm2, %xmm2 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm4 +; AVX1-NEXT: vpand %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vpsllw $5, %xmm8, %xmm4 +; AVX1-NEXT: vpblendvb %xmm4, %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $2, %xmm0, %xmm3 +; AVX1-NEXT: vpand %xmm6, %xmm3, %xmm3 +; AVX1-NEXT: vpaddb %xmm4, %xmm4, %xmm4 +; AVX1-NEXT: vpblendvb %xmm4, %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $1, %xmm0, %xmm3 +; AVX1-NEXT: vpand %xmm7, %xmm3, %xmm3 +; AVX1-NEXT: vpaddb %xmm4, %xmm4, %xmm4 +; AVX1-NEXT: vpblendvb %xmm4, %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: var_rotate_v32i8: +; AVX2: # BB#0: +; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX2-NEXT: vpsubb %ymm1, %ymm2, %ymm2 +; AVX2-NEXT: vpsllw $5, %ymm1, %ymm1 +; AVX2-NEXT: vpsllw $4, %ymm0, %ymm3 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm3, %ymm3 +; AVX2-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm3 +; AVX2-NEXT: vpsllw $2, %ymm3, %ymm4 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm4, %ymm4 +; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vpblendvb %ymm1, %ymm4, %ymm3, %ymm3 +; AVX2-NEXT: vpaddb %ymm3, %ymm3, %ymm4 +; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vpblendvb %ymm1, %ymm4, %ymm3, %ymm1 +; AVX2-NEXT: vpsllw $5, %ymm2, %ymm2 +; AVX2-NEXT: vpaddb %ymm2, %ymm2, %ymm3 +; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm4 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm4, %ymm4 +; AVX2-NEXT: vpblendvb %ymm2, %ymm4, %ymm0, %ymm0 +; AVX2-NEXT: vpsrlw $2, %ymm0, %ymm2 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX2-NEXT: vpblendvb %ymm3, %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpsrlw $1, %ymm0, %ymm2 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX2-NEXT: vpaddb %ymm3, %ymm3, %ymm3 +; AVX2-NEXT: vpblendvb %ymm3, %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_rotate_v32i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; XOPAVX1-NEXT: vprotb %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vprotb %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_rotate_v32i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm3 +; XOPAVX2-NEXT: vprotb %xmm2, %xmm3, %xmm2 +; XOPAVX2-NEXT: vprotb %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq + %b8 = sub <32 x i8> , %b + %shl = shl <32 x i8> %a, %b + %lshr = lshr <32 x i8> %a, %b8 + %or = or <32 x i8> %shl, %lshr + ret <32 x i8> %or +} + +; +; Constant Rotates +; + +define <4 x i64> @constant_rotate_v4i64(<4 x i64> %a) nounwind { +; AVX1-LABEL: constant_rotate_v4i64: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpsllq $60, %xmm1, %xmm2 +; AVX1-NEXT: vpsllq $50, %xmm1, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vpsllq $14, %xmm0, %xmm3 +; AVX1-NEXT: vpsllq $4, %xmm0, %xmm4 +; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7] +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vpsrlq $2, %xmm1, %xmm3 +; AVX1-NEXT: vpsrlq $14, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5,6,7] +; AVX1-NEXT: vpsrlq $50, %xmm0, %xmm3 +; AVX1-NEXT: vpsrlq $60, %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7] +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vorps %ymm0, %ymm2, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: constant_rotate_v4i64: +; AVX2: # BB#0: +; AVX2-NEXT: vpsllvq {{.*}}(%rip), %ymm0, %ymm1 +; AVX2-NEXT: vpsrlvq {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_rotate_v4i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpshlq {{.*}}(%rip), %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpshlq {{.*}}(%rip), %xmm2, %xmm3 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1 +; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX1-NEXT: vpsubq {{.*}}(%rip), %xmm3, %xmm4 +; XOPAVX1-NEXT: vpshlq %xmm4, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpsubq {{.*}}(%rip), %xmm3, %xmm3 +; XOPAVX1-NEXT: vpshlq %xmm3, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_rotate_v4i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsllvq {{.*}}(%rip), %ymm0, %ymm1 +; XOPAVX2-NEXT: vpsrlvq {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; XOPAVX2-NEXT: retq + %shl = shl <4 x i64> %a, + %lshr = lshr <4 x i64> %a, + %or = or <4 x i64> %shl, %lshr + ret <4 x i64> %or +} + +define <8 x i32> @constant_rotate_v8i32(<8 x i32> %a) nounwind { +; AVX1-LABEL: constant_rotate_v8i32: +; AVX1: # BB#0: +; AVX1-NEXT: vpmulld {{.*}}(%rip), %xmm0, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpmulld {{.*}}(%rip), %xmm2, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1 +; AVX1-NEXT: vpsrld $21, %xmm2, %xmm3 +; AVX1-NEXT: vpsrld $23, %xmm2, %xmm4 +; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7] +; AVX1-NEXT: vpsrld $22, %xmm2, %xmm4 +; AVX1-NEXT: vpsrld $24, %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm4[4,5,6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7] +; AVX1-NEXT: vpsrld $25, %xmm0, %xmm3 +; AVX1-NEXT: vpsrld $27, %xmm0, %xmm4 +; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7] +; AVX1-NEXT: vpsrld $26, %xmm0, %xmm4 +; AVX1-NEXT: vpsrld $28, %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4,5,6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7] +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: constant_rotate_v8i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm1 +; AVX2-NEXT: vpsrlvd {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_rotate_v8i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm2, %xmm3 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1 +; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm2, %xmm2 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_rotate_v8i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm1 +; XOPAVX2-NEXT: vpsrlvd {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; XOPAVX2-NEXT: retq + %shl = shl <8 x i32> %a, + %lshr = lshr <8 x i32> %a, + %or = or <8 x i32> %shl, %lshr + ret <8 x i32> %or +} + +define <16 x i16> @constant_rotate_v8i16(<16 x i16> %a) nounwind { +; AVX1-LABEL: constant_rotate_v8i16: +; AVX1: # BB#0: +; AVX1-NEXT: vpmullw {{.*}}(%rip), %xmm0, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpmullw {{.*}}(%rip), %xmm2, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1 +; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [32896,28784,24672,20560,16448,12336,8224,4112] +; AVX1-NEXT: vpblendvb %xmm4, %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpsrlw $4, %xmm2, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [256,57568,49344,41120,32896,24672,16448,8224] +; AVX1-NEXT: vpblendvb %xmm4, %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpsrlw $2, %xmm2, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [512,49600,33152,16704,256,49344,32896,16448] +; AVX1-NEXT: vpblendvb %xmm4, %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpsrlw $1, %xmm2, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [1024,33664,768,33408,512,33152,256,32896] +; AVX1-NEXT: vpblendvb %xmm4, %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [256,61680,57568,53456,49344,45232,41120,37008] +; AVX1-NEXT: vpblendvb %xmm4, %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [512,57824,49600,41376,33152,24928,16704,8480] +; AVX1-NEXT: vpblendvb %xmm4, %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $2, %xmm0, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [1024,50112,33664,17216,768,49856,33408,16960] +; AVX1-NEXT: vpblendvb %xmm4, %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $1, %xmm0, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [2048,34688,1792,34432,1536,34176,1280,33920] +; AVX1-NEXT: vpblendvb %xmm4, %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: constant_rotate_v8i16: +; AVX2: # BB#0: +; AVX2-NEXT: vpmullw {{.*}}(%rip), %ymm0, %ymm1 +; AVX2-NEXT: vpxor %ymm2, %ymm2, %ymm2 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1] +; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[12],ymm2[12],ymm3[13],ymm2[13],ymm3[14],ymm2[14],ymm3[15],ymm2[15] +; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15] +; AVX2-NEXT: vpsrlvd %ymm4, %ymm5, %ymm4 +; AVX2-NEXT: vpsrld $16, %ymm4, %ymm4 +; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11] +; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11] +; AVX2-NEXT: vpsrlvd %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0 +; AVX2-NEXT: vpackusdw %ymm4, %ymm0, %ymm0 +; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_rotate_v8i16: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpshlw {{.*}}(%rip), %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpshlw {{.*}}(%rip), %xmm2, %xmm3 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1 +; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX1-NEXT: vpsubw {{.*}}(%rip), %xmm3, %xmm4 +; XOPAVX1-NEXT: vpshlw %xmm4, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpsubw {{.*}}(%rip), %xmm3, %xmm3 +; XOPAVX1-NEXT: vpshlw %xmm3, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_rotate_v8i16: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpmullw {{.*}}(%rip), %ymm0, %ymm1 +; XOPAVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX2-NEXT: vpsubw {{.*}}(%rip), %xmm2, %xmm3 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm4 +; XOPAVX2-NEXT: vpshlw %xmm3, %xmm4, %xmm3 +; XOPAVX2-NEXT: vpsubw {{.*}}(%rip), %xmm2, %xmm2 +; XOPAVX2-NEXT: vpshlw %xmm2, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0 +; XOPAVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; XOPAVX2-NEXT: retq + %shl = shl <16 x i16> %a, + %lshr = lshr <16 x i16> %a, + %or = or <16 x i16> %shl, %lshr + ret <16 x i16> %or +} + +define <32 x i8> @constant_rotate_v32i8(<32 x i8> %a) nounwind { +; AVX1-LABEL: constant_rotate_v32i8: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpsllw $4, %xmm1, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm8 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240] +; AVX1-NEXT: vpand %xmm8, %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,2,3,4,5,6,7,8,7,6,5,4,3,2,1] +; AVX1-NEXT: vpsllw $5, %xmm4, %xmm4 +; AVX1-NEXT: vpblendvb %xmm4, %xmm2, %xmm1, %xmm2 +; AVX1-NEXT: vpsllw $2, %xmm2, %xmm5 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252] +; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm5 +; AVX1-NEXT: vpaddb %xmm4, %xmm4, %xmm7 +; AVX1-NEXT: vpblendvb %xmm7, %xmm5, %xmm2, %xmm2 +; AVX1-NEXT: vpaddb %xmm2, %xmm2, %xmm5 +; AVX1-NEXT: vpaddb %xmm7, %xmm7, %xmm3 +; AVX1-NEXT: vpblendvb %xmm3, %xmm5, %xmm2, %xmm2 +; AVX1-NEXT: vpsllw $4, %xmm0, %xmm5 +; AVX1-NEXT: vpand %xmm8, %xmm5, %xmm5 +; AVX1-NEXT: vpblendvb %xmm4, %xmm5, %xmm0, %xmm4 +; AVX1-NEXT: vpsllw $2, %xmm4, %xmm5 +; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm5 +; AVX1-NEXT: vpblendvb %xmm7, %xmm5, %xmm4, %xmm4 +; AVX1-NEXT: vpaddb %xmm4, %xmm4, %xmm5 +; AVX1-NEXT: vpblendvb %xmm3, %xmm5, %xmm4, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm9 +; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm8 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vpand %xmm8, %xmm3, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [8,7,6,5,4,3,2,1,0,1,2,3,4,5,6,7] +; AVX1-NEXT: vpsllw $5, %xmm5, %xmm5 +; AVX1-NEXT: vpblendvb %xmm5, %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpsrlw $2, %xmm1, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63] +; AVX1-NEXT: vpand %xmm6, %xmm3, %xmm3 +; AVX1-NEXT: vpaddb %xmm5, %xmm5, %xmm7 +; AVX1-NEXT: vpblendvb %xmm7, %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpsrlw $1, %xmm1, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127] +; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vpaddb %xmm7, %xmm7, %xmm2 +; AVX1-NEXT: vpblendvb %xmm2, %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm3 +; AVX1-NEXT: vpand %xmm8, %xmm3, %xmm3 +; AVX1-NEXT: vpblendvb %xmm5, %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $2, %xmm0, %xmm3 +; AVX1-NEXT: vpand %xmm6, %xmm3, %xmm3 +; AVX1-NEXT: vpblendvb %xmm7, %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $1, %xmm0, %xmm3 +; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vpblendvb %xmm2, %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vorps %ymm0, %ymm9, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: constant_rotate_v32i8: +; AVX2: # BB#0: +; AVX2-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,8,7,6,5,4,3,2,1,0,1,2,3,4,5,6,7,8,7,6,5,4,3,2,1] +; AVX2-NEXT: vpsllw $5, %ymm1, %ymm1 +; AVX2-NEXT: vpsllw $4, %ymm0, %ymm2 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm2 +; AVX2-NEXT: vpsllw $2, %ymm2, %ymm3 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm3, %ymm3 +; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vpblendvb %ymm1, %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vpaddb %ymm2, %ymm2, %ymm3 +; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vpblendvb %ymm1, %ymm3, %ymm2, %ymm1 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [8,7,6,5,4,3,2,1,0,1,2,3,4,5,6,7,8,7,6,5,4,3,2,1,0,1,2,3,4,5,6,7] +; AVX2-NEXT: vpsllw $5, %ymm2, %ymm2 +; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm3 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm3, %ymm3 +; AVX2-NEXT: vpblendvb %ymm2, %ymm3, %ymm0, %ymm0 +; AVX2-NEXT: vpsrlw $2, %ymm0, %ymm3 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm3, %ymm3 +; AVX2-NEXT: vpaddb %ymm2, %ymm2, %ymm2 +; AVX2-NEXT: vpblendvb %ymm2, %ymm3, %ymm0, %ymm0 +; AVX2-NEXT: vpsrlw $1, %ymm0, %ymm3 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm3, %ymm3 +; AVX2-NEXT: vpaddb %ymm2, %ymm2, %ymm2 +; AVX2-NEXT: vpblendvb %ymm2, %ymm3, %ymm0, %ymm0 +; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_rotate_v32i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,8,7,6,5,4,3,2,1] +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpshlb %xmm1, %xmm2, %xmm3 +; XOPAVX1-NEXT: vpshlb %xmm1, %xmm0, %xmm1 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1 +; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX1-NEXT: vpsubb {{.*}}(%rip), %xmm3, %xmm3 +; XOPAVX1-NEXT: vpshlb %xmm3, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpshlb %xmm3, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_rotate_v32i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,8,7,6,5,4,3,2,1] +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm2 +; XOPAVX2-NEXT: vpshlb %xmm1, %xmm2, %xmm3 +; XOPAVX2-NEXT: vpshlb %xmm1, %xmm0, %xmm1 +; XOPAVX2-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1 +; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX2-NEXT: vpsubb {{.*}}(%rip), %xmm3, %xmm3 +; XOPAVX2-NEXT: vpshlb %xmm3, %xmm2, %xmm2 +; XOPAVX2-NEXT: vpshlb %xmm3, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; XOPAVX2-NEXT: retq + %shl = shl <32 x i8> %a, + %lshr = lshr <32 x i8> %a, + %or = or <32 x i8> %shl, %lshr + ret <32 x i8> %or +} + +; +; Uniform Constant Rotates +; + +define <4 x i64> @splatconstant_rotate_v4i64(<4 x i64> %a) nounwind { +; AVX1-LABEL: splatconstant_rotate_v4i64: +; AVX1: # BB#0: +; AVX1-NEXT: vpsllq $14, %xmm0, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpsllq $14, %xmm2, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1 +; AVX1-NEXT: vpsrlq $50, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlq $50, %xmm2, %xmm2 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: splatconstant_rotate_v4i64: +; AVX2: # BB#0: +; AVX2-NEXT: vpsllq $14, %ymm0, %ymm1 +; AVX2-NEXT: vpsrlq $50, %ymm0, %ymm0 +; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_rotate_v4i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vprotq $14, %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vprotq $14, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_rotate_v4i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vprotq $14, %xmm0, %xmm1 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 +; XOPAVX2-NEXT: vprotq $14, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX2-NEXT: retq + %shl = shl <4 x i64> %a, + %lshr = lshr <4 x i64> %a, + %or = or <4 x i64> %shl, %lshr + ret <4 x i64> %or +} + +define <8 x i32> @splatconstant_rotate_v8i32(<8 x i32> %a) nounwind { +; AVX1-LABEL: splatconstant_rotate_v8i32: +; AVX1: # BB#0: +; AVX1-NEXT: vpslld $4, %xmm0, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpslld $4, %xmm2, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1 +; AVX1-NEXT: vpsrld $28, %xmm0, %xmm0 +; AVX1-NEXT: vpsrld $28, %xmm2, %xmm2 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: splatconstant_rotate_v8i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpslld $4, %ymm0, %ymm1 +; AVX2-NEXT: vpsrld $28, %ymm0, %ymm0 +; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_rotate_v8i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vprotd $4, %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vprotd $4, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_rotate_v8i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vprotd $4, %xmm0, %xmm1 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 +; XOPAVX2-NEXT: vprotd $4, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX2-NEXT: retq + %shl = shl <8 x i32> %a, + %lshr = lshr <8 x i32> %a, + %or = or <8 x i32> %shl, %lshr + ret <8 x i32> %or +} + +define <16 x i16> @splatconstant_rotate_v16i16(<16 x i16> %a) nounwind { +; AVX1-LABEL: splatconstant_rotate_v16i16: +; AVX1: # BB#0: +; AVX1-NEXT: vpsllw $7, %xmm0, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpsllw $7, %xmm2, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1 +; AVX1-NEXT: vpsrlw $9, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $9, %xmm2, %xmm2 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: splatconstant_rotate_v16i16: +; AVX2: # BB#0: +; AVX2-NEXT: vpsllw $7, %ymm0, %ymm1 +; AVX2-NEXT: vpsrlw $9, %ymm0, %ymm0 +; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_rotate_v16i16: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vprotw $7, %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vprotw $7, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_rotate_v16i16: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vprotw $7, %xmm0, %xmm1 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 +; XOPAVX2-NEXT: vprotw $7, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX2-NEXT: retq + %shl = shl <16 x i16> %a, + %lshr = lshr <16 x i16> %a, + %or = or <16 x i16> %shl, %lshr + ret <16 x i16> %or +} + +define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind { +; AVX1-LABEL: splatconstant_rotate_v32i8: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpsllw $4, %xmm1, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240] +; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpsllw $4, %xmm0, %xmm4 +; AVX1-NEXT: vpand %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vorps %ymm0, %ymm2, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: splatconstant_rotate_v32i8: +; AVX2: # BB#0: +; AVX2-NEXT: vpsllw $4, %ymm0, %ymm1 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm1, %ymm1 +; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_rotate_v32i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vprotb $4, %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vprotb $4, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_rotate_v32i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vprotb $4, %xmm0, %xmm1 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 +; XOPAVX2-NEXT: vprotb $4, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX2-NEXT: retq + %shl = shl <32 x i8> %a, + %lshr = lshr <32 x i8> %a, + %or = or <32 x i8> %shl, %lshr + ret <32 x i8> %or +} + +; +; Masked Uniform Constant Rotates +; + +define <4 x i64> @splatconstant_rotate_mask_v4i64(<4 x i64> %a) nounwind { +; AVX1-LABEL: splatconstant_rotate_mask_v4i64: +; AVX1: # BB#0: +; AVX1-NEXT: vpsllq $15, %xmm0, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpsllq $15, %xmm2, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1 +; AVX1-NEXT: vpsrlq $49, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlq $49, %xmm2, %xmm2 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: vandps {{.*}}(%rip), %ymm0, %ymm0 +; AVX1-NEXT: vandps {{.*}}(%rip), %ymm1, %ymm1 +; AVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: splatconstant_rotate_mask_v4i64: +; AVX2: # BB#0: +; AVX2-NEXT: vpsllq $15, %ymm0, %ymm1 +; AVX2-NEXT: vpsrlq $49, %ymm0, %ymm0 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm1, %ymm1 +; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_rotate_mask_v4i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vprotq $15, %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vprotq $15, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: vandps {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_rotate_mask_v4i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vprotq $15, %xmm0, %xmm1 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 +; XOPAVX2-NEXT: vprotq $15, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX2-NEXT: retq + %shl = shl <4 x i64> %a, + %lshr = lshr <4 x i64> %a, + %rmask = and <4 x i64> %lshr, + %lmask = and <4 x i64> %shl, + %or = or <4 x i64> %lmask, %rmask + ret <4 x i64> %or +} + +define <8 x i32> @splatconstant_rotate_mask_v8i32(<8 x i32> %a) nounwind { +; AVX1-LABEL: splatconstant_rotate_mask_v8i32: +; AVX1: # BB#0: +; AVX1-NEXT: vpslld $4, %xmm0, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpslld $4, %xmm2, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1 +; AVX1-NEXT: vpsrld $28, %xmm0, %xmm0 +; AVX1-NEXT: vpsrld $28, %xmm2, %xmm2 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: vandps {{.*}}(%rip), %ymm0, %ymm0 +; AVX1-NEXT: vandps {{.*}}(%rip), %ymm1, %ymm1 +; AVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: splatconstant_rotate_mask_v8i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpslld $4, %ymm0, %ymm1 +; AVX2-NEXT: vpsrld $28, %ymm0, %ymm0 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm1, %ymm1 +; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_rotate_mask_v8i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vprotd $4, %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vprotd $4, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: vandps {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_rotate_mask_v8i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vprotd $4, %xmm0, %xmm1 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 +; XOPAVX2-NEXT: vprotd $4, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX2-NEXT: retq + %shl = shl <8 x i32> %a, + %lshr = lshr <8 x i32> %a, + %rmask = and <8 x i32> %lshr, + %lmask = and <8 x i32> %shl, + %or = or <8 x i32> %lmask, %rmask + ret <8 x i32> %or +} + +define <16 x i16> @splatconstant_rotate_mask_v16i16(<16 x i16> %a) nounwind { +; AVX1-LABEL: splatconstant_rotate_mask_v16i16: +; AVX1: # BB#0: +; AVX1-NEXT: vpsllw $5, %xmm0, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpsllw $5, %xmm2, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1 +; AVX1-NEXT: vpsrlw $11, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlw $11, %xmm2, %xmm2 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: vandps {{.*}}(%rip), %ymm0, %ymm0 +; AVX1-NEXT: vandps {{.*}}(%rip), %ymm1, %ymm1 +; AVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: splatconstant_rotate_mask_v16i16: +; AVX2: # BB#0: +; AVX2-NEXT: vpsllw $5, %ymm0, %ymm1 +; AVX2-NEXT: vpsrlw $11, %ymm0, %ymm0 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm1, %ymm1 +; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_rotate_mask_v16i16: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vprotw $5, %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vprotw $5, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: vandps {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_rotate_mask_v16i16: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vprotw $5, %xmm0, %xmm1 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 +; XOPAVX2-NEXT: vprotw $5, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX2-NEXT: retq + %shl = shl <16 x i16> %a, + %lshr = lshr <16 x i16> %a, + %rmask = and <16 x i16> %lshr, + %lmask = and <16 x i16> %shl, + %or = or <16 x i16> %lmask, %rmask + ret <16 x i16> %or +} + +define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind { +; AVX1-LABEL: splatconstant_rotate_mask_v32i8: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpsllw $4, %xmm1, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240] +; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpsllw $4, %xmm0, %xmm4 +; AVX1-NEXT: vpand %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vandps {{.*}}(%rip), %ymm0, %ymm0 +; AVX1-NEXT: vandps {{.*}}(%rip), %ymm2, %ymm1 +; AVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: splatconstant_rotate_mask_v32i8: +; AVX2: # BB#0: +; AVX2-NEXT: vpsllw $4, %ymm0, %ymm1 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm1, %ymm1 +; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm1, %ymm1 +; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_rotate_mask_v32i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vprotb $4, %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vprotb $4, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: vandps {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_rotate_mask_v32i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vprotb $4, %xmm0, %xmm1 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 +; XOPAVX2-NEXT: vprotb $4, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX2-NEXT: retq + %shl = shl <32 x i8> %a, + %lshr = lshr <32 x i8> %a, + %rmask = and <32 x i8> %lshr, + %lmask = and <32 x i8> %shl, + %or = or <32 x i8> %lmask, %rmask + ret <32 x i8> %or +} diff --git a/test/CodeGen/X86/vector-sext.ll b/test/CodeGen/X86/vector-sext.ll index 8e79493ddd07..b63c3f084b22 100644 --- a/test/CodeGen/X86/vector-sext.ll +++ b/test/CodeGen/X86/vector-sext.ll @@ -1,396 +1,3813 @@ -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=SSE --check-prefix=SSSE3 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2 +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefix=SSE --check-prefix=SSSE3 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2 ; ; Just one 32-bit run to make sure we do reasonable things there. -; RUN: llc < %s -mtriple=i686-unknown-unknown -mcpu=i686 -mattr=+sse4.1 | FileCheck %s --check-prefix=X32-SSE41 +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=X32-SSE41 -define <8 x i32> @sext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp { -; SSE2-LABEL: sext_8i16_to_8i32: +define <8 x i16> @sext_16i8_to_8i16(<16 x i8> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: sext_16i8_to_8i16: ; SSE2: # BB#0: # %entry -; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] -; SSE2-NEXT: psrad $16, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] -; SSE2-NEXT: psrad $16, %xmm1 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: psraw $8, %xmm0 +; SSE2-NEXT: retq +; +; SSSE3-LABEL: sext_16i8_to_8i16: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSSE3-NEXT: psraw $8, %xmm0 +; SSSE3-NEXT: retq +; +; SSE41-LABEL: sext_16i8_to_8i16: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovsxbw %xmm0, %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: sext_16i8_to_8i16: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovsxbw %xmm0, %xmm0 +; AVX-NEXT: retq +; +; X32-SSE41-LABEL: sext_16i8_to_8i16: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: pmovsxbw %xmm0, %xmm0 +; X32-SSE41-NEXT: retl +entry: + %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> + %C = sext <8 x i8> %B to <8 x i16> + ret <8 x i16> %C +} + +define <16 x i16> @sext_16i8_to_16i16(<16 x i8> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: sext_16i8_to_16i16: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] +; SSE2-NEXT: psraw $8, %xmm2 +; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] +; SSE2-NEXT: psraw $8, %xmm1 ; SSE2-NEXT: movdqa %xmm2, %xmm0 ; SSE2-NEXT: retq ; -; SSSE3-LABEL: sext_8i16_to_8i32: +; SSSE3-LABEL: sext_16i8_to_16i16: ; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] -; SSSE3-NEXT: psrad $16, %xmm2 -; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] -; SSSE3-NEXT: psrad $16, %xmm1 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] +; SSSE3-NEXT: psraw $8, %xmm2 +; SSSE3-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] +; SSSE3-NEXT: psraw $8, %xmm1 ; SSSE3-NEXT: movdqa %xmm2, %xmm0 ; SSSE3-NEXT: retq ; -; SSE41-LABEL: sext_8i16_to_8i32: +; SSE41-LABEL: sext_16i8_to_16i16: ; SSE41: # BB#0: # %entry -; SSE41-NEXT: pmovsxwd %xmm0, %xmm2 +; SSE41-NEXT: pmovsxbw %xmm0, %xmm2 ; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE41-NEXT: pmovsxwd %xmm0, %xmm1 +; SSE41-NEXT: pmovsxbw %xmm0, %xmm1 ; SSE41-NEXT: movdqa %xmm2, %xmm0 ; SSE41-NEXT: retq ; -; AVX1-LABEL: sext_8i16_to_8i32: +; AVX1-LABEL: sext_16i8_to_16i16: ; AVX1: # BB#0: # %entry -; AVX1-NEXT: vpmovsxwd %xmm0, %xmm1 +; AVX1-NEXT: vpmovsxbw %xmm0, %xmm1 ; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; AVX1-NEXT: vpmovsxwd %xmm0, %xmm0 +; AVX1-NEXT: vpmovsxbw %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 ; AVX1-NEXT: retq ; -; AVX2-LABEL: sext_8i16_to_8i32: +; AVX2-LABEL: sext_16i8_to_16i16: ; AVX2: # BB#0: # %entry -; AVX2-NEXT: vpmovsxwd %xmm0, %ymm0 +; AVX2-NEXT: vpmovsxbw %xmm0, %ymm0 ; AVX2-NEXT: retq ; -; X32-SSE41-LABEL: sext_8i16_to_8i32: +; X32-SSE41-LABEL: sext_16i8_to_16i16: ; X32-SSE41: # BB#0: # %entry -; X32-SSE41-NEXT: pmovsxwd %xmm0, %xmm2 +; X32-SSE41-NEXT: pmovsxbw %xmm0, %xmm2 ; X32-SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; X32-SSE41-NEXT: pmovsxwd %xmm0, %xmm1 +; X32-SSE41-NEXT: pmovsxbw %xmm0, %xmm1 ; X32-SSE41-NEXT: movdqa %xmm2, %xmm0 ; X32-SSE41-NEXT: retl entry: - %B = sext <8 x i16> %A to <8 x i32> - ret <8 x i32>%B + %B = sext <16 x i8> %A to <16 x i16> + ret <16 x i16> %B } -define <4 x i64> @sext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp { -; SSE2-LABEL: sext_4i32_to_4i64: +define <4 x i32> @sext_16i8_to_4i32(<16 x i8> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: sext_16i8_to_4i32: ; SSE2: # BB#0: # %entry -; SSE2-NEXT: movdqa %xmm0, %xmm2 -; SSE2-NEXT: psrad $31, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] -; SSE2-NEXT: movdqa %xmm1, %xmm2 -; SSE2-NEXT: psrad $31, %xmm2 -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE2-NEXT: psrad $24, %xmm0 ; SSE2-NEXT: retq ; -; SSSE3-LABEL: sext_4i32_to_4i64: +; SSSE3-LABEL: sext_16i8_to_4i32: ; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: movdqa %xmm0, %xmm2 -; SSSE3-NEXT: psrad $31, %xmm2 -; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] -; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] -; SSSE3-NEXT: movdqa %xmm1, %xmm2 -; SSSE3-NEXT: psrad $31, %xmm2 -; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSSE3-NEXT: psrad $24, %xmm0 ; SSSE3-NEXT: retq ; -; SSE41-LABEL: sext_4i32_to_4i64: +; SSE41-LABEL: sext_16i8_to_4i32: ; SSE41: # BB#0: # %entry -; SSE41-NEXT: pmovsxdq %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE41-NEXT: pmovsxdq %xmm0, %xmm1 +; SSE41-NEXT: pmovsxbd %xmm0, %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: sext_16i8_to_4i32: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovsxbd %xmm0, %xmm0 +; AVX-NEXT: retq +; +; X32-SSE41-LABEL: sext_16i8_to_4i32: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: pmovsxbd %xmm0, %xmm0 +; X32-SSE41-NEXT: retl +entry: + %B = shufflevector <16 x i8> %A, <16 x i8> undef, <4 x i32> + %C = sext <4 x i8> %B to <4 x i32> + ret <4 x i32> %C +} + +define <8 x i32> @sext_16i8_to_8i32(<16 x i8> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: sext_16i8_to_8i32: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSE2-NEXT: psrad $24, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSE2-NEXT: psrad $24, %xmm1 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSSE3-LABEL: sext_16i8_to_8i32: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSSE3-NEXT: psrad $24, %xmm2 +; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSSE3-NEXT: psrad $24, %xmm1 +; SSSE3-NEXT: movdqa %xmm2, %xmm0 +; SSSE3-NEXT: retq +; +; SSE41-LABEL: sext_16i8_to_8i32: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovsxbd %xmm0, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; SSE41-NEXT: pmovsxbd %xmm0, %xmm1 ; SSE41-NEXT: movdqa %xmm2, %xmm0 ; SSE41-NEXT: retq ; -; AVX1-LABEL: sext_4i32_to_4i64: +; AVX1-LABEL: sext_16i8_to_8i32: ; AVX1: # BB#0: # %entry -; AVX1-NEXT: vpmovsxdq %xmm0, %xmm1 -; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; AVX1-NEXT: vpmovsxdq %xmm0, %xmm0 +; AVX1-NEXT: vpmovsxbd %xmm0, %xmm1 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; AVX1-NEXT: vpmovsxbd %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 ; AVX1-NEXT: retq ; -; AVX2-LABEL: sext_4i32_to_4i64: +; AVX2-LABEL: sext_16i8_to_8i32: ; AVX2: # BB#0: # %entry -; AVX2-NEXT: vpmovsxdq %xmm0, %ymm0 +; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero +; AVX2-NEXT: vpslld $24, %ymm0, %ymm0 +; AVX2-NEXT: vpsrad $24, %ymm0, %ymm0 ; AVX2-NEXT: retq ; -; X32-SSE41-LABEL: sext_4i32_to_4i64: +; X32-SSE41-LABEL: sext_16i8_to_8i32: ; X32-SSE41: # BB#0: # %entry -; X32-SSE41-NEXT: pmovsxdq %xmm0, %xmm2 -; X32-SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; X32-SSE41-NEXT: pmovsxdq %xmm0, %xmm1 +; X32-SSE41-NEXT: pmovsxbd %xmm0, %xmm2 +; X32-SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; X32-SSE41-NEXT: pmovsxbd %xmm0, %xmm1 ; X32-SSE41-NEXT: movdqa %xmm2, %xmm0 ; X32-SSE41-NEXT: retl entry: - %B = sext <4 x i32> %A to <4 x i64> - ret <4 x i64>%B + %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> + %C = sext <8 x i8> %B to <8 x i32> + ret <8 x i32> %C } -define i32 @sext_2i8_to_i32(<16 x i8> %A) nounwind uwtable readnone ssp { -; SSE2-LABEL: sext_2i8_to_i32: +define <2 x i64> @sext_16i8_to_2i64(<16 x i8> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: sext_16i8_to_2i64: ; SSE2: # BB#0: # %entry ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE2-NEXT: psraw $8, %xmm0 -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: psrad $31, %xmm1 +; SSE2-NEXT: psrad $24, %xmm0 +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; SSE2-NEXT: retq ; -; SSSE3-LABEL: sext_2i8_to_i32: +; SSSE3-LABEL: sext_16i8_to_2i64: ; SSSE3: # BB#0: # %entry ; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSSE3-NEXT: psraw $8, %xmm0 -; SSSE3-NEXT: movd %xmm0, %eax +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: psrad $31, %xmm1 +; SSSE3-NEXT: psrad $24, %xmm0 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; SSSE3-NEXT: retq ; -; SSE41-LABEL: sext_2i8_to_i32: +; SSE41-LABEL: sext_16i8_to_2i64: ; SSE41: # BB#0: # %entry -; SSE41-NEXT: pmovsxbw %xmm0, %xmm0 -; SSE41-NEXT: movd %xmm0, %eax +; SSE41-NEXT: pmovsxbq %xmm0, %xmm0 ; SSE41-NEXT: retq ; -; AVX-LABEL: sext_2i8_to_i32: +; AVX-LABEL: sext_16i8_to_2i64: ; AVX: # BB#0: # %entry -; AVX-NEXT: vpmovsxbw %xmm0, %xmm0 -; AVX-NEXT: vmovd %xmm0, %eax +; AVX-NEXT: vpmovsxbq %xmm0, %xmm0 ; AVX-NEXT: retq ; -; X32-SSE41-LABEL: sext_2i8_to_i32: +; X32-SSE41-LABEL: sext_16i8_to_2i64: ; X32-SSE41: # BB#0: # %entry -; X32-SSE41: pmovsxbw %xmm0, %xmm0 -; X32-SSE41-NEXT: movd %xmm0, %eax -; X32-SSE41-NEXT: popl %edx +; X32-SSE41-NEXT: pmovsxbq %xmm0, %xmm0 ; X32-SSE41-NEXT: retl entry: - %Shuf = shufflevector <16 x i8> %A, <16 x i8> undef, <2 x i32> - %Ex = sext <2 x i8> %Shuf to <2 x i16> - %Bc = bitcast <2 x i16> %Ex to i32 - ret i32 %Bc + %B = shufflevector <16 x i8> %A, <16 x i8> undef, <2 x i32> + %C = sext <2 x i8> %B to <2 x i64> + ret <2 x i64> %C } -define <4 x i32> @load_sext_test1(<4 x i16> *%ptr) { -; SSE2-LABEL: load_sext_test1: +define <4 x i64> @sext_16i8_to_4i64(<16 x i8> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: sext_16i8_to_4i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSE2-NEXT: movdqa %xmm2, %xmm1 +; SSE2-NEXT: psrad $31, %xmm1 +; SSE2-NEXT: psrad $24, %xmm2 +; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE2-NEXT: psrld $16, %xmm0 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: psrad $31, %xmm0 +; SSE2-NEXT: psrad $24, %xmm1 +; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSSE3-LABEL: sext_16i8_to_4i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSSE3-NEXT: movdqa %xmm2, %xmm1 +; SSSE3-NEXT: psrad $31, %xmm1 +; SSSE3-NEXT: psrad $24, %xmm2 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSSE3-NEXT: psrld $16, %xmm0 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: psrad $31, %xmm0 +; SSSE3-NEXT: psrad $24, %xmm1 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; SSSE3-NEXT: movdqa %xmm2, %xmm0 +; SSSE3-NEXT: retq +; +; SSE41-LABEL: sext_16i8_to_4i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovsxbq %xmm0, %xmm2 +; SSE41-NEXT: psrld $16, %xmm0 +; SSE41-NEXT: pmovsxbq %xmm0, %xmm1 +; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: retq +; +; AVX1-LABEL: sext_16i8_to_4i64: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpmovsxbq %xmm0, %xmm1 +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX1-NEXT: vpmovsxbq %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: sext_16i8_to_4i64: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX2-NEXT: vpslld $24, %xmm0, %xmm0 +; AVX2-NEXT: vpsrad $24, %xmm0, %xmm0 +; AVX2-NEXT: vpmovsxdq %xmm0, %ymm0 +; AVX2-NEXT: retq +; +; X32-SSE41-LABEL: sext_16i8_to_4i64: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: pmovsxbq %xmm0, %xmm2 +; X32-SSE41-NEXT: psrld $16, %xmm0 +; X32-SSE41-NEXT: pmovsxbq %xmm0, %xmm1 +; X32-SSE41-NEXT: movdqa %xmm2, %xmm0 +; X32-SSE41-NEXT: retl +entry: + %B = shufflevector <16 x i8> %A, <16 x i8> undef, <4 x i32> + %C = sext <4 x i8> %B to <4 x i64> + ret <4 x i64> %C +} + +define <4 x i32> @sext_8i16_to_4i32(<8 x i16> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: sext_8i16_to_4i32: ; SSE2: # BB#0: # %entry -; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero ; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] ; SSE2-NEXT: psrad $16, %xmm0 ; SSE2-NEXT: retq ; -; SSSE3-LABEL: load_sext_test1: +; SSSE3-LABEL: sext_8i16_to_4i32: ; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: movq {{.*#+}} xmm0 = mem[0],zero ; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] ; SSSE3-NEXT: psrad $16, %xmm0 ; SSSE3-NEXT: retq ; -; SSE41-LABEL: load_sext_test1: +; SSE41-LABEL: sext_8i16_to_4i32: ; SSE41: # BB#0: # %entry -; SSE41-NEXT: pmovsxwd (%rdi), %xmm0 +; SSE41-NEXT: pmovsxwd %xmm0, %xmm0 ; SSE41-NEXT: retq ; -; AVX-LABEL: load_sext_test1: +; AVX-LABEL: sext_8i16_to_4i32: ; AVX: # BB#0: # %entry -; AVX-NEXT: vpmovsxwd (%rdi), %xmm0 +; AVX-NEXT: vpmovsxwd %xmm0, %xmm0 ; AVX-NEXT: retq ; -; X32-SSE41-LABEL: load_sext_test1: +; X32-SSE41-LABEL: sext_8i16_to_4i32: ; X32-SSE41: # BB#0: # %entry -; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-SSE41-NEXT: pmovsxwd (%eax), %xmm0 +; X32-SSE41-NEXT: pmovsxwd %xmm0, %xmm0 ; X32-SSE41-NEXT: retl entry: - %X = load <4 x i16>, <4 x i16>* %ptr - %Y = sext <4 x i16> %X to <4 x i32> - ret <4 x i32>%Y + %B = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> + %C = sext <4 x i16> %B to <4 x i32> + ret <4 x i32> %C } -define <4 x i32> @load_sext_test2(<4 x i8> *%ptr) { -; SSE2-LABEL: load_sext_test2: +define <8 x i32> @sext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: sext_8i16_to_8i32: ; SSE2: # BB#0: # %entry -; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] -; SSE2-NEXT: psrad $24, %xmm0 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] +; SSE2-NEXT: psrad $16, %xmm2 +; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSE2-NEXT: psrad $16, %xmm1 +; SSE2-NEXT: movdqa %xmm2, %xmm0 ; SSE2-NEXT: retq ; -; SSSE3-LABEL: load_sext_test2: +; SSSE3-LABEL: sext_8i16_to_8i32: ; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] -; SSSE3-NEXT: psrad $24, %xmm0 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] +; SSSE3-NEXT: psrad $16, %xmm2 +; SSSE3-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSSE3-NEXT: psrad $16, %xmm1 +; SSSE3-NEXT: movdqa %xmm2, %xmm0 ; SSSE3-NEXT: retq ; -; SSE41-LABEL: load_sext_test2: +; SSE41-LABEL: sext_8i16_to_8i32: ; SSE41: # BB#0: # %entry -; SSE41-NEXT: pmovsxbd (%rdi), %xmm0 +; SSE41-NEXT: pmovsxwd %xmm0, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; SSE41-NEXT: pmovsxwd %xmm0, %xmm1 +; SSE41-NEXT: movdqa %xmm2, %xmm0 ; SSE41-NEXT: retq ; -; AVX-LABEL: load_sext_test2: -; AVX: # BB#0: # %entry -; AVX-NEXT: vpmovsxbd (%rdi), %xmm0 -; AVX-NEXT: retq +; AVX1-LABEL: sext_8i16_to_8i32: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpmovsxwd %xmm0, %xmm1 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; AVX1-NEXT: vpmovsxwd %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: sext_8i16_to_8i32: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovsxwd %xmm0, %ymm0 +; AVX2-NEXT: retq ; -; X32-SSE41-LABEL: load_sext_test2: +; X32-SSE41-LABEL: sext_8i16_to_8i32: ; X32-SSE41: # BB#0: # %entry -; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-SSE41-NEXT: pmovsxbd (%eax), %xmm0 +; X32-SSE41-NEXT: pmovsxwd %xmm0, %xmm2 +; X32-SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; X32-SSE41-NEXT: pmovsxwd %xmm0, %xmm1 +; X32-SSE41-NEXT: movdqa %xmm2, %xmm0 ; X32-SSE41-NEXT: retl entry: - %X = load <4 x i8>, <4 x i8>* %ptr - %Y = sext <4 x i8> %X to <4 x i32> - ret <4 x i32>%Y + %B = sext <8 x i16> %A to <8 x i32> + ret <8 x i32> %B } -define <2 x i64> @load_sext_test3(<2 x i8> *%ptr) { -; SSE2-LABEL: load_sext_test3: +define <2 x i64> @sext_8i16_to_2i64(<8 x i16> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: sext_8i16_to_2i64: ; SSE2: # BB#0: # %entry -; SSE2-NEXT: movzwl (%rdi), %eax -; SSE2-NEXT: movd %eax, %xmm0 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] ; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] ; SSE2-NEXT: movdqa %xmm0, %xmm1 ; SSE2-NEXT: psrad $31, %xmm1 -; SSE2-NEXT: psrad $24, %xmm0 +; SSE2-NEXT: psrad $16, %xmm0 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; SSE2-NEXT: retq ; -; SSSE3-LABEL: load_sext_test3: +; SSSE3-LABEL: sext_8i16_to_2i64: ; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: movzwl (%rdi), %eax -; SSSE3-NEXT: movd %eax, %xmm0 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] ; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] ; SSSE3-NEXT: movdqa %xmm0, %xmm1 ; SSSE3-NEXT: psrad $31, %xmm1 -; SSSE3-NEXT: psrad $24, %xmm0 +; SSSE3-NEXT: psrad $16, %xmm0 ; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; SSSE3-NEXT: retq ; -; SSE41-LABEL: load_sext_test3: +; SSE41-LABEL: sext_8i16_to_2i64: ; SSE41: # BB#0: # %entry -; SSE41-NEXT: pmovsxbq (%rdi), %xmm0 +; SSE41-NEXT: pmovsxwq %xmm0, %xmm0 ; SSE41-NEXT: retq ; -; AVX-LABEL: load_sext_test3: -; AVX: # BB#0: # %entry -; AVX-NEXT: vpmovsxbq (%rdi), %xmm0 -; AVX-NEXT: retq +; AVX-LABEL: sext_8i16_to_2i64: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovsxwq %xmm0, %xmm0 +; AVX-NEXT: retq +; +; X32-SSE41-LABEL: sext_8i16_to_2i64: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: pmovsxwq %xmm0, %xmm0 +; X32-SSE41-NEXT: retl +entry: + %B = shufflevector <8 x i16> %A, <8 x i16> undef, <2 x i32> + %C = sext <2 x i16> %B to <2 x i64> + ret <2 x i64> %C +} + +define <4 x i64> @sext_8i16_to_4i64(<8 x i16> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: sext_8i16_to_4i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] +; SSE2-NEXT: movdqa %xmm2, %xmm1 +; SSE2-NEXT: psrad $31, %xmm1 +; SSE2-NEXT: psrad $16, %xmm2 +; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: psrad $31, %xmm0 +; SSE2-NEXT: psrad $16, %xmm1 +; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSSE3-LABEL: sext_8i16_to_4i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] +; SSSE3-NEXT: movdqa %xmm2, %xmm1 +; SSSE3-NEXT: psrad $31, %xmm1 +; SSSE3-NEXT: psrad $16, %xmm2 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: psrad $31, %xmm0 +; SSSE3-NEXT: psrad $16, %xmm1 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; SSSE3-NEXT: movdqa %xmm2, %xmm0 +; SSSE3-NEXT: retq +; +; SSE41-LABEL: sext_8i16_to_4i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovsxwq %xmm0, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; SSE41-NEXT: pmovsxwq %xmm0, %xmm1 +; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: retq +; +; AVX1-LABEL: sext_8i16_to_4i64: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpmovsxwq %xmm0, %xmm1 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; AVX1-NEXT: vpmovsxwq %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: sext_8i16_to_4i64: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX2-NEXT: vpslld $16, %xmm0, %xmm0 +; AVX2-NEXT: vpsrad $16, %xmm0, %xmm0 +; AVX2-NEXT: vpmovsxdq %xmm0, %ymm0 +; AVX2-NEXT: retq +; +; X32-SSE41-LABEL: sext_8i16_to_4i64: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: pmovsxwq %xmm0, %xmm2 +; X32-SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; X32-SSE41-NEXT: pmovsxwq %xmm0, %xmm1 +; X32-SSE41-NEXT: movdqa %xmm2, %xmm0 +; X32-SSE41-NEXT: retl +entry: + %B = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> + %C = sext <4 x i16> %B to <4 x i64> + ret <4 x i64> %C +} + +define <2 x i64> @sext_4i32_to_2i64(<4 x i32> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: sext_4i32_to_2i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: psrad $31, %xmm1 +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: sext_4i32_to_2i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: psrad $31, %xmm1 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: sext_4i32_to_2i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovsxdq %xmm0, %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: sext_4i32_to_2i64: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovsxdq %xmm0, %xmm0 +; AVX-NEXT: retq +; +; X32-SSE41-LABEL: sext_4i32_to_2i64: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: pmovsxdq %xmm0, %xmm0 +; X32-SSE41-NEXT: retl +entry: + %B = shufflevector <4 x i32> %A, <4 x i32> undef, <2 x i32> + %C = sext <2 x i32> %B to <2 x i64> + ret <2 x i64> %C +} + +define <4 x i64> @sext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: sext_4i32_to_4i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: psrad $31, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: psrad $31, %xmm2 +; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: sext_4i32_to_4i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movdqa %xmm0, %xmm2 +; SSSE3-NEXT: psrad $31, %xmm2 +; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSSE3-NEXT: movdqa %xmm1, %xmm2 +; SSSE3-NEXT: psrad $31, %xmm2 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: sext_4i32_to_4i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovsxdq %xmm0, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; SSE41-NEXT: pmovsxdq %xmm0, %xmm1 +; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: retq +; +; AVX1-LABEL: sext_4i32_to_4i64: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpmovsxdq %xmm0, %xmm1 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; AVX1-NEXT: vpmovsxdq %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: sext_4i32_to_4i64: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovsxdq %xmm0, %ymm0 +; AVX2-NEXT: retq +; +; X32-SSE41-LABEL: sext_4i32_to_4i64: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: pmovsxdq %xmm0, %xmm2 +; X32-SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; X32-SSE41-NEXT: pmovsxdq %xmm0, %xmm1 +; X32-SSE41-NEXT: movdqa %xmm2, %xmm0 +; X32-SSE41-NEXT: retl +entry: + %B = sext <4 x i32> %A to <4 x i64> + ret <4 x i64> %B +} + +define <2 x i64> @load_sext_2i1_to_2i64(<2 x i1> *%ptr) { +; SSE-LABEL: load_sext_2i1_to_2i64: +; SSE: # BB#0: # %entry +; SSE-NEXT: movzbl (%rdi), %eax +; SSE-NEXT: movq %rax, %rcx +; SSE-NEXT: shlq $62, %rcx +; SSE-NEXT: sarq $63, %rcx +; SSE-NEXT: movd %rcx, %xmm1 +; SSE-NEXT: shlq $63, %rax +; SSE-NEXT: sarq $63, %rax +; SSE-NEXT: movd %rax, %xmm0 +; SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE-NEXT: retq +; +; AVX-LABEL: load_sext_2i1_to_2i64: +; AVX: # BB#0: # %entry +; AVX-NEXT: movzbl (%rdi), %eax +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $62, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vmovq %rcx, %xmm0 +; AVX-NEXT: shlq $63, %rax +; AVX-NEXT: sarq $63, %rax +; AVX-NEXT: vmovq %rax, %xmm1 +; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_2i1_to_2i64: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-SSE41-NEXT: movzbl (%eax), %eax +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $31, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: movd %ecx, %xmm0 +; X32-SSE41-NEXT: pinsrd $1, %ecx, %xmm0 +; X32-SSE41-NEXT: shll $30, %eax +; X32-SSE41-NEXT: sarl $31, %eax +; X32-SSE41-NEXT: pinsrd $2, %eax, %xmm0 +; X32-SSE41-NEXT: pinsrd $3, %eax, %xmm0 +; X32-SSE41-NEXT: retl +entry: + %X = load <2 x i1>, <2 x i1>* %ptr + %Y = sext <2 x i1> %X to <2 x i64> + ret <2 x i64> %Y +} + +define <2 x i64> @load_sext_2i8_to_2i64(<2 x i8> *%ptr) { +; SSE2-LABEL: load_sext_2i8_to_2i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movzwl (%rdi), %eax +; SSE2-NEXT: movd %eax, %xmm0 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: psrad $31, %xmm1 +; SSE2-NEXT: psrad $24, %xmm0 +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_sext_2i8_to_2i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movzwl (%rdi), %eax +; SSSE3-NEXT: movd %eax, %xmm0 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: psrad $31, %xmm1 +; SSSE3-NEXT: psrad $24, %xmm0 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_sext_2i8_to_2i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovsxbq (%rdi), %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: load_sext_2i8_to_2i64: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovsxbq (%rdi), %xmm0 +; AVX-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_2i8_to_2i64: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-SSE41-NEXT: pmovsxbq (%eax), %xmm0 +; X32-SSE41-NEXT: retl +entry: + %X = load <2 x i8>, <2 x i8>* %ptr + %Y = sext <2 x i8> %X to <2 x i64> + ret <2 x i64> %Y +} + +define <4 x i32> @load_sext_4i1_to_4i32(<4 x i1> *%ptr) { +; SSE2-LABEL: load_sext_4i1_to_4i32: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movzbl (%rdi), %eax +; SSE2-NEXT: movq %rax, %rcx +; SSE2-NEXT: shlq $60, %rcx +; SSE2-NEXT: sarq $63, %rcx +; SSE2-NEXT: movd %ecx, %xmm0 +; SSE2-NEXT: movq %rax, %rcx +; SSE2-NEXT: shlq $62, %rcx +; SSE2-NEXT: sarq $63, %rcx +; SSE2-NEXT: movd %ecx, %xmm1 +; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; SSE2-NEXT: movq %rax, %rcx +; SSE2-NEXT: shlq $61, %rcx +; SSE2-NEXT: sarq $63, %rcx +; SSE2-NEXT: movd %ecx, %xmm2 +; SSE2-NEXT: shlq $63, %rax +; SSE2-NEXT: sarq $63, %rax +; SSE2-NEXT: movd %eax, %xmm0 +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_sext_4i1_to_4i32: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movzbl (%rdi), %eax +; SSSE3-NEXT: movq %rax, %rcx +; SSSE3-NEXT: shlq $60, %rcx +; SSSE3-NEXT: sarq $63, %rcx +; SSSE3-NEXT: movd %ecx, %xmm0 +; SSSE3-NEXT: movq %rax, %rcx +; SSSE3-NEXT: shlq $62, %rcx +; SSSE3-NEXT: sarq $63, %rcx +; SSSE3-NEXT: movd %ecx, %xmm1 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; SSSE3-NEXT: movq %rax, %rcx +; SSSE3-NEXT: shlq $61, %rcx +; SSSE3-NEXT: sarq $63, %rcx +; SSSE3-NEXT: movd %ecx, %xmm2 +; SSSE3-NEXT: shlq $63, %rax +; SSSE3-NEXT: sarq $63, %rax +; SSSE3-NEXT: movd %eax, %xmm0 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_sext_4i1_to_4i32: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: movzbl (%rdi), %eax +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $62, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: movq %rax, %rdx +; SSE41-NEXT: shlq $63, %rdx +; SSE41-NEXT: sarq $63, %rdx +; SSE41-NEXT: movd %edx, %xmm0 +; SSE41-NEXT: pinsrd $1, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $61, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrd $2, %ecx, %xmm0 +; SSE41-NEXT: shlq $60, %rax +; SSE41-NEXT: sarq $63, %rax +; SSE41-NEXT: pinsrd $3, %eax, %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: load_sext_4i1_to_4i32: +; AVX: # BB#0: # %entry +; AVX-NEXT: movzbl (%rdi), %eax +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $62, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: movq %rax, %rdx +; AVX-NEXT: shlq $63, %rdx +; AVX-NEXT: sarq $63, %rdx +; AVX-NEXT: vmovd %edx, %xmm0 +; AVX-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $61, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0 +; AVX-NEXT: shlq $60, %rax +; AVX-NEXT: sarq $63, %rax +; AVX-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_4i1_to_4i32: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-SSE41-NEXT: movl (%eax), %eax +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $30, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: movl %eax, %edx +; X32-SSE41-NEXT: shll $31, %edx +; X32-SSE41-NEXT: sarl $31, %edx +; X32-SSE41-NEXT: movd %edx, %xmm0 +; X32-SSE41-NEXT: pinsrd $1, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $29, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrd $2, %ecx, %xmm0 +; X32-SSE41-NEXT: shll $28, %eax +; X32-SSE41-NEXT: sarl $31, %eax +; X32-SSE41-NEXT: pinsrd $3, %eax, %xmm0 +; X32-SSE41-NEXT: retl +entry: + %X = load <4 x i1>, <4 x i1>* %ptr + %Y = sext <4 x i1> %X to <4 x i32> + ret <4 x i32> %Y +} + +define <4 x i32> @load_sext_4i8_to_4i32(<4 x i8> *%ptr) { +; SSE2-LABEL: load_sext_4i8_to_4i32: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE2-NEXT: psrad $24, %xmm0 +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_sext_4i8_to_4i32: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSSE3-NEXT: psrad $24, %xmm0 +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_sext_4i8_to_4i32: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovsxbd (%rdi), %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: load_sext_4i8_to_4i32: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovsxbd (%rdi), %xmm0 +; AVX-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_4i8_to_4i32: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-SSE41-NEXT: pmovsxbd (%eax), %xmm0 +; X32-SSE41-NEXT: retl +entry: + %X = load <4 x i8>, <4 x i8>* %ptr + %Y = sext <4 x i8> %X to <4 x i32> + ret <4 x i32> %Y +} + +define <4 x i64> @load_sext_4i1_to_4i64(<4 x i1> *%ptr) { +; SSE2-LABEL: load_sext_4i1_to_4i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movzbl (%rdi), %eax +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $3, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm0 +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm1 +; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm2 +; SSE2-NEXT: shrl $2, %eax +; SSE2-NEXT: andl $1, %eax +; SSE2-NEXT: movd %eax, %xmm0 +; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] +; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,1,1,3] +; SSE2-NEXT: psllq $63, %xmm0 +; SSE2-NEXT: psrad $31, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3] +; SSE2-NEXT: psllq $63, %xmm1 +; SSE2-NEXT: psrad $31, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_sext_4i1_to_4i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movzbl (%rdi), %eax +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $3, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm0 +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm1 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm2 +; SSSE3-NEXT: shrl $2, %eax +; SSSE3-NEXT: andl $1, %eax +; SSSE3-NEXT: movd %eax, %xmm0 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] +; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,1,1,3] +; SSSE3-NEXT: psllq $63, %xmm0 +; SSSE3-NEXT: psrad $31, %xmm0 +; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3] +; SSSE3-NEXT: psllq $63, %xmm1 +; SSSE3-NEXT: psrad $31, %xmm1 +; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_sext_4i1_to_4i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: movzbl (%rdi), %eax +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: movl %eax, %edx +; SSE41-NEXT: andl $1, %edx +; SSE41-NEXT: movd %edx, %xmm1 +; SSE41-NEXT: pinsrd $1, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $2, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrd $2, %ecx, %xmm1 +; SSE41-NEXT: shrl $3, %eax +; SSE41-NEXT: andl $1, %eax +; SSE41-NEXT: pinsrd $3, %eax, %xmm1 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero +; SSE41-NEXT: psllq $63, %xmm0 +; SSE41-NEXT: psrad $31, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3] +; SSE41-NEXT: psllq $63, %xmm1 +; SSE41-NEXT: psrad $31, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; SSE41-NEXT: retq +; +; AVX1-LABEL: load_sext_4i1_to_4i64: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: movzbl (%rdi), %eax +; AVX1-NEXT: movq %rax, %rcx +; AVX1-NEXT: shlq $62, %rcx +; AVX1-NEXT: sarq $63, %rcx +; AVX1-NEXT: movq %rax, %rdx +; AVX1-NEXT: shlq $63, %rdx +; AVX1-NEXT: sarq $63, %rdx +; AVX1-NEXT: vmovd %edx, %xmm0 +; AVX1-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %rcx +; AVX1-NEXT: shlq $61, %rcx +; AVX1-NEXT: sarq $63, %rcx +; AVX1-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0 +; AVX1-NEXT: shlq $60, %rax +; AVX1-NEXT: sarq $63, %rax +; AVX1-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0 +; AVX1-NEXT: vpmovsxdq %xmm0, %xmm1 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; AVX1-NEXT: vpmovsxdq %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: load_sext_4i1_to_4i64: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: movzbl (%rdi), %eax +; AVX2-NEXT: movq %rax, %rcx +; AVX2-NEXT: shlq $60, %rcx +; AVX2-NEXT: sarq $63, %rcx +; AVX2-NEXT: vmovq %rcx, %xmm0 +; AVX2-NEXT: movq %rax, %rcx +; AVX2-NEXT: shlq $61, %rcx +; AVX2-NEXT: sarq $63, %rcx +; AVX2-NEXT: vmovq %rcx, %xmm1 +; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX2-NEXT: movq %rax, %rcx +; AVX2-NEXT: shlq $62, %rcx +; AVX2-NEXT: sarq $63, %rcx +; AVX2-NEXT: vmovq %rcx, %xmm1 +; AVX2-NEXT: shlq $63, %rax +; AVX2-NEXT: sarq $63, %rax +; AVX2-NEXT: vmovq %rax, %xmm2 +; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_4i1_to_4i64: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-SSE41-NEXT: movzbl (%eax), %eax +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: movl %eax, %edx +; X32-SSE41-NEXT: andl $1, %edx +; X32-SSE41-NEXT: movd %edx, %xmm1 +; X32-SSE41-NEXT: pinsrd $1, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $2, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrd $2, %ecx, %xmm1 +; X32-SSE41-NEXT: shrl $3, %eax +; X32-SSE41-NEXT: andl $1, %eax +; X32-SSE41-NEXT: pinsrd $3, %eax, %xmm1 +; X32-SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero +; X32-SSE41-NEXT: psllq $63, %xmm0 +; X32-SSE41-NEXT: psrad $31, %xmm0 +; X32-SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; X32-SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3] +; X32-SSE41-NEXT: psllq $63, %xmm1 +; X32-SSE41-NEXT: psrad $31, %xmm1 +; X32-SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; X32-SSE41-NEXT: retl +entry: + %X = load <4 x i1>, <4 x i1>* %ptr + %Y = sext <4 x i1> %X to <4 x i64> + ret <4 x i64> %Y +} + +define <4 x i64> @load_sext_4i8_to_4i64(<4 x i8> *%ptr) { +; SSE2-LABEL: load_sext_4i8_to_4i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movsbq 1(%rdi), %rax +; SSE2-NEXT: movd %rax, %xmm1 +; SSE2-NEXT: movsbq (%rdi), %rax +; SSE2-NEXT: movd %rax, %xmm0 +; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE2-NEXT: movsbq 3(%rdi), %rax +; SSE2-NEXT: movd %rax, %xmm2 +; SSE2-NEXT: movsbq 2(%rdi), %rax +; SSE2-NEXT: movd %rax, %xmm1 +; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_sext_4i8_to_4i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movsbq 1(%rdi), %rax +; SSSE3-NEXT: movd %rax, %xmm1 +; SSSE3-NEXT: movsbq (%rdi), %rax +; SSSE3-NEXT: movd %rax, %xmm0 +; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSSE3-NEXT: movsbq 3(%rdi), %rax +; SSSE3-NEXT: movd %rax, %xmm2 +; SSSE3-NEXT: movsbq 2(%rdi), %rax +; SSSE3-NEXT: movd %rax, %xmm1 +; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_sext_4i8_to_4i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovsxbq (%rdi), %xmm0 +; SSE41-NEXT: pmovsxbq 2(%rdi), %xmm1 +; SSE41-NEXT: retq +; +; AVX1-LABEL: load_sext_4i8_to_4i64: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpmovsxbd (%rdi), %xmm0 +; AVX1-NEXT: vpmovsxdq %xmm0, %xmm1 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; AVX1-NEXT: vpmovsxdq %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: load_sext_4i8_to_4i64: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovsxbq (%rdi), %ymm0 +; AVX2-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_4i8_to_4i64: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-SSE41-NEXT: pmovsxbq (%eax), %xmm0 +; X32-SSE41-NEXT: pmovsxbq 2(%eax), %xmm1 +; X32-SSE41-NEXT: retl +entry: + %X = load <4 x i8>, <4 x i8>* %ptr + %Y = sext <4 x i8> %X to <4 x i64> + ret <4 x i64> %Y +} + +define <8 x i16> @load_sext_8i1_to_8i16(<8 x i1> *%ptr) { +; SSE2-LABEL: load_sext_8i1_to_8i16: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movsbq (%rdi), %rax +; SSE2-NEXT: movq %rax, %rcx +; SSE2-NEXT: shrq $7, %rcx +; SSE2-NEXT: movd %ecx, %xmm0 +; SSE2-NEXT: movq %rax, %rcx +; SSE2-NEXT: shlq $60, %rcx +; SSE2-NEXT: sarq $63, %rcx +; SSE2-NEXT: movd %ecx, %xmm2 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] +; SSE2-NEXT: movq %rax, %rcx +; SSE2-NEXT: shlq $58, %rcx +; SSE2-NEXT: sarq $63, %rcx +; SSE2-NEXT: movd %ecx, %xmm0 +; SSE2-NEXT: movq %rax, %rcx +; SSE2-NEXT: shlq $62, %rcx +; SSE2-NEXT: sarq $63, %rcx +; SSE2-NEXT: movd %ecx, %xmm1 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSE2-NEXT: movq %rax, %rcx +; SSE2-NEXT: shlq $57, %rcx +; SSE2-NEXT: sarq $63, %rcx +; SSE2-NEXT: movd %ecx, %xmm0 +; SSE2-NEXT: movq %rax, %rcx +; SSE2-NEXT: shlq $61, %rcx +; SSE2-NEXT: sarq $63, %rcx +; SSE2-NEXT: movd %ecx, %xmm2 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] +; SSE2-NEXT: movq %rax, %rcx +; SSE2-NEXT: shlq $59, %rcx +; SSE2-NEXT: sarq $63, %rcx +; SSE2-NEXT: movd %ecx, %xmm3 +; SSE2-NEXT: shlq $63, %rax +; SSE2-NEXT: sarq $63, %rax +; SSE2-NEXT: movd %eax, %xmm0 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_sext_8i1_to_8i16: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movsbq (%rdi), %rax +; SSSE3-NEXT: movq %rax, %rcx +; SSSE3-NEXT: shrq $7, %rcx +; SSSE3-NEXT: movd %ecx, %xmm0 +; SSSE3-NEXT: movq %rax, %rcx +; SSSE3-NEXT: shlq $60, %rcx +; SSSE3-NEXT: sarq $63, %rcx +; SSSE3-NEXT: movd %ecx, %xmm2 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] +; SSSE3-NEXT: movq %rax, %rcx +; SSSE3-NEXT: shlq $58, %rcx +; SSSE3-NEXT: sarq $63, %rcx +; SSSE3-NEXT: movd %ecx, %xmm0 +; SSSE3-NEXT: movq %rax, %rcx +; SSSE3-NEXT: shlq $62, %rcx +; SSSE3-NEXT: sarq $63, %rcx +; SSSE3-NEXT: movd %ecx, %xmm1 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSSE3-NEXT: movq %rax, %rcx +; SSSE3-NEXT: shlq $57, %rcx +; SSSE3-NEXT: sarq $63, %rcx +; SSSE3-NEXT: movd %ecx, %xmm0 +; SSSE3-NEXT: movq %rax, %rcx +; SSSE3-NEXT: shlq $61, %rcx +; SSSE3-NEXT: sarq $63, %rcx +; SSSE3-NEXT: movd %ecx, %xmm2 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] +; SSSE3-NEXT: movq %rax, %rcx +; SSSE3-NEXT: shlq $59, %rcx +; SSSE3-NEXT: sarq $63, %rcx +; SSSE3-NEXT: movd %ecx, %xmm3 +; SSSE3-NEXT: shlq $63, %rax +; SSSE3-NEXT: sarq $63, %rax +; SSSE3-NEXT: movd %eax, %xmm0 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_sext_8i1_to_8i16: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: movsbq (%rdi), %rax +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $62, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: movq %rax, %rdx +; SSE41-NEXT: shlq $63, %rdx +; SSE41-NEXT: sarq $63, %rdx +; SSE41-NEXT: movd %edx, %xmm0 +; SSE41-NEXT: pinsrw $1, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $61, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrw $2, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $60, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrw $3, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $59, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrw $4, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $58, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrw $5, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $57, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrw $6, %ecx, %xmm0 +; SSE41-NEXT: shrq $7, %rax +; SSE41-NEXT: pinsrw $7, %eax, %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: load_sext_8i1_to_8i16: +; AVX: # BB#0: # %entry +; AVX-NEXT: movsbq (%rdi), %rax +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $62, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: movq %rax, %rdx +; AVX-NEXT: shlq $63, %rdx +; AVX-NEXT: sarq $63, %rdx +; AVX-NEXT: vmovd %edx, %xmm0 +; AVX-NEXT: vpinsrw $1, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $61, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrw $2, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $60, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrw $3, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $59, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrw $4, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $58, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrw $5, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $57, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0 +; AVX-NEXT: shrq $7, %rax +; AVX-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_8i1_to_8i16: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-SSE41-NEXT: movsbl (%eax), %eax +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $30, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: movl %eax, %edx +; X32-SSE41-NEXT: shll $31, %edx +; X32-SSE41-NEXT: sarl $31, %edx +; X32-SSE41-NEXT: movd %edx, %xmm0 +; X32-SSE41-NEXT: pinsrw $1, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $29, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrw $2, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $28, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrw $3, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $27, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrw $4, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $26, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrw $5, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $25, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrw $6, %ecx, %xmm0 +; X32-SSE41-NEXT: shrl $7, %eax +; X32-SSE41-NEXT: pinsrw $7, %eax, %xmm0 +; X32-SSE41-NEXT: retl +entry: + %X = load <8 x i1>, <8 x i1>* %ptr + %Y = sext <8 x i1> %X to <8 x i16> + ret <8 x i16> %Y +} + +define <8 x i16> @load_sext_8i8_to_8i16(<8 x i8> *%ptr) { +; SSE2-LABEL: load_sext_8i8_to_8i16: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: psraw $8, %xmm0 +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_sext_8i8_to_8i16: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSSE3-NEXT: psraw $8, %xmm0 +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_sext_8i8_to_8i16: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovsxbw (%rdi), %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: load_sext_8i8_to_8i16: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovsxbw (%rdi), %xmm0 +; AVX-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_8i8_to_8i16: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-SSE41-NEXT: pmovsxbw (%eax), %xmm0 +; X32-SSE41-NEXT: retl +entry: + %X = load <8 x i8>, <8 x i8>* %ptr + %Y = sext <8 x i8> %X to <8 x i16> + ret <8 x i16> %Y +} + +define <8 x i32> @load_sext_8i1_to_8i32(<8 x i1> *%ptr) { +; SSE2-LABEL: load_sext_8i1_to_8i32: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movzbl (%rdi), %eax +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $6, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm0 +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $2, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm2 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm1 +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $4, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm0 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $5, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm0 +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm2 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $3, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm0 +; SSE2-NEXT: shrl $7, %eax +; SSE2-NEXT: movzwl %ax, %eax +; SSE2-NEXT: movd %eax, %xmm3 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE2-NEXT: pslld $31, %xmm0 +; SSE2-NEXT: psrad $31, %xmm0 +; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSE2-NEXT: pslld $31, %xmm1 +; SSE2-NEXT: psrad $31, %xmm1 +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_sext_8i1_to_8i32: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movzbl (%rdi), %eax +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $6, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm0 +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $2, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm2 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm1 +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $4, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm0 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $5, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm0 +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm2 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $3, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm0 +; SSSE3-NEXT: shrl $7, %eax +; SSSE3-NEXT: movzwl %ax, %eax +; SSSE3-NEXT: movd %eax, %xmm3 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSSE3-NEXT: pslld $31, %xmm0 +; SSSE3-NEXT: psrad $31, %xmm0 +; SSSE3-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSSE3-NEXT: pslld $31, %xmm1 +; SSSE3-NEXT: psrad $31, %xmm1 +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_sext_8i1_to_8i32: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: movzbl (%rdi), %eax +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: movl %eax, %edx +; SSE41-NEXT: andl $1, %edx +; SSE41-NEXT: movd %edx, %xmm1 +; SSE41-NEXT: pinsrw $1, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $2, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrw $2, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $3, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrw $3, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $4, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrw $4, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $5, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrw $5, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $6, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrw $6, %ecx, %xmm1 +; SSE41-NEXT: shrl $7, %eax +; SSE41-NEXT: movzwl %ax, %eax +; SSE41-NEXT: pinsrw $7, %eax, %xmm1 +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero +; SSE41-NEXT: pslld $31, %xmm0 +; SSE41-NEXT: psrad $31, %xmm0 +; SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSE41-NEXT: pslld $31, %xmm1 +; SSE41-NEXT: psrad $31, %xmm1 +; SSE41-NEXT: retq +; +; AVX1-LABEL: load_sext_8i1_to_8i32: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: movsbq (%rdi), %rax +; AVX1-NEXT: movq %rax, %rcx +; AVX1-NEXT: shlq $58, %rcx +; AVX1-NEXT: sarq $63, %rcx +; AVX1-NEXT: movq %rax, %rdx +; AVX1-NEXT: shlq $59, %rdx +; AVX1-NEXT: sarq $63, %rdx +; AVX1-NEXT: vmovd %edx, %xmm0 +; AVX1-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %rcx +; AVX1-NEXT: shlq $57, %rcx +; AVX1-NEXT: sarq $63, %rcx +; AVX1-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %rcx +; AVX1-NEXT: shrq $7, %rcx +; AVX1-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %rcx +; AVX1-NEXT: shlq $62, %rcx +; AVX1-NEXT: sarq $63, %rcx +; AVX1-NEXT: movq %rax, %rdx +; AVX1-NEXT: shlq $63, %rdx +; AVX1-NEXT: sarq $63, %rdx +; AVX1-NEXT: vmovd %edx, %xmm1 +; AVX1-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm1 +; AVX1-NEXT: movq %rax, %rcx +; AVX1-NEXT: shlq $61, %rcx +; AVX1-NEXT: sarq $63, %rcx +; AVX1-NEXT: vpinsrd $2, %ecx, %xmm1, %xmm1 +; AVX1-NEXT: shlq $60, %rax +; AVX1-NEXT: sarq $63, %rax +; AVX1-NEXT: vpinsrd $3, %eax, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: load_sext_8i1_to_8i32: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: movsbq (%rdi), %rax +; AVX2-NEXT: movq %rax, %rcx +; AVX2-NEXT: shlq $58, %rcx +; AVX2-NEXT: sarq $63, %rcx +; AVX2-NEXT: movq %rax, %rdx +; AVX2-NEXT: shlq $59, %rdx +; AVX2-NEXT: sarq $63, %rdx +; AVX2-NEXT: vmovd %edx, %xmm0 +; AVX2-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0 +; AVX2-NEXT: movq %rax, %rcx +; AVX2-NEXT: shlq $57, %rcx +; AVX2-NEXT: sarq $63, %rcx +; AVX2-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0 +; AVX2-NEXT: movq %rax, %rcx +; AVX2-NEXT: shrq $7, %rcx +; AVX2-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0 +; AVX2-NEXT: movq %rax, %rcx +; AVX2-NEXT: shlq $62, %rcx +; AVX2-NEXT: sarq $63, %rcx +; AVX2-NEXT: movq %rax, %rdx +; AVX2-NEXT: shlq $63, %rdx +; AVX2-NEXT: sarq $63, %rdx +; AVX2-NEXT: vmovd %edx, %xmm1 +; AVX2-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm1 +; AVX2-NEXT: movq %rax, %rcx +; AVX2-NEXT: shlq $61, %rcx +; AVX2-NEXT: sarq $63, %rcx +; AVX2-NEXT: vpinsrd $2, %ecx, %xmm1, %xmm1 +; AVX2-NEXT: shlq $60, %rax +; AVX2-NEXT: sarq $63, %rax +; AVX2-NEXT: vpinsrd $3, %eax, %xmm1, %xmm1 +; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_8i1_to_8i32: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-SSE41-NEXT: movzbl (%eax), %eax +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: movl %eax, %edx +; X32-SSE41-NEXT: andl $1, %edx +; X32-SSE41-NEXT: movd %edx, %xmm1 +; X32-SSE41-NEXT: pinsrw $1, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $2, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrw $2, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $3, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrw $3, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $4, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrw $4, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $5, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrw $5, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $6, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrw $6, %ecx, %xmm1 +; X32-SSE41-NEXT: shrl $7, %eax +; X32-SSE41-NEXT: pinsrw $7, %eax, %xmm1 +; X32-SSE41-NEXT: pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero +; X32-SSE41-NEXT: pslld $31, %xmm0 +; X32-SSE41-NEXT: psrad $31, %xmm0 +; X32-SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; X32-SSE41-NEXT: pslld $31, %xmm1 +; X32-SSE41-NEXT: psrad $31, %xmm1 +; X32-SSE41-NEXT: retl +entry: + %X = load <8 x i1>, <8 x i1>* %ptr + %Y = sext <8 x i1> %X to <8 x i32> + ret <8 x i32> %Y +} + +define <8 x i32> @load_sext_8i8_to_8i32(<8 x i8> *%ptr) { +; SSE2-LABEL: load_sext_8i8_to_8i32: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE2-NEXT: psrad $24, %xmm0 +; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] +; SSE2-NEXT: psrad $24, %xmm1 +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_sext_8i8_to_8i32: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSSE3-NEXT: psrad $24, %xmm0 +; SSSE3-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] +; SSSE3-NEXT: psrad $24, %xmm1 +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_sext_8i8_to_8i32: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovsxbd (%rdi), %xmm0 +; SSE41-NEXT: pmovsxbd 4(%rdi), %xmm1 +; SSE41-NEXT: retq +; +; AVX1-LABEL: load_sext_8i8_to_8i32: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpmovsxbw (%rdi), %xmm0 +; AVX1-NEXT: vpmovsxwd %xmm0, %xmm1 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; AVX1-NEXT: vpmovsxwd %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: load_sext_8i8_to_8i32: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovsxbd (%rdi), %ymm0 +; AVX2-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_8i8_to_8i32: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-SSE41-NEXT: pmovsxbd (%eax), %xmm0 +; X32-SSE41-NEXT: pmovsxbd 4(%eax), %xmm1 +; X32-SSE41-NEXT: retl +entry: + %X = load <8 x i8>, <8 x i8>* %ptr + %Y = sext <8 x i8> %X to <8 x i32> + ret <8 x i32> %Y +} + +define <16 x i8> @load_sext_16i1_to_16i8(<16 x i1> *%ptr) nounwind readnone { +; SSE2-LABEL: load_sext_16i1_to_16i8: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: pushq %rbp +; SSE2-NEXT: pushq %r15 +; SSE2-NEXT: pushq %r14 +; SSE2-NEXT: pushq %r13 +; SSE2-NEXT: pushq %r12 +; SSE2-NEXT: pushq %rbx +; SSE2-NEXT: movswq (%rdi), %rax +; SSE2-NEXT: movq %rax, %r8 +; SSE2-NEXT: movq %rax, %r9 +; SSE2-NEXT: movq %rax, %r10 +; SSE2-NEXT: movq %rax, %r11 +; SSE2-NEXT: movq %rax, %r14 +; SSE2-NEXT: movq %rax, %r15 +; SSE2-NEXT: movq %rax, %r12 +; SSE2-NEXT: movq %rax, %r13 +; SSE2-NEXT: movq %rax, %rbx +; SSE2-NEXT: movq %rax, %rcx +; SSE2-NEXT: movq %rax, %rdx +; SSE2-NEXT: movq %rax, %rsi +; SSE2-NEXT: movq %rax, %rdi +; SSE2-NEXT: movq %rax, %rbp +; SSE2-NEXT: shlq $49, %rbp +; SSE2-NEXT: sarq $63, %rbp +; SSE2-NEXT: movd %ebp, %xmm0 +; SSE2-NEXT: movq %rax, %rbp +; SSE2-NEXT: movsbq %al, %rax +; SSE2-NEXT: shlq $57, %r8 +; SSE2-NEXT: sarq $63, %r8 +; SSE2-NEXT: movd %r8d, %xmm1 +; SSE2-NEXT: shlq $53, %r9 +; SSE2-NEXT: sarq $63, %r9 +; SSE2-NEXT: movd %r9d, %xmm2 +; SSE2-NEXT: shlq $61, %r10 +; SSE2-NEXT: sarq $63, %r10 +; SSE2-NEXT: movd %r10d, %xmm3 +; SSE2-NEXT: shlq $51, %r11 +; SSE2-NEXT: sarq $63, %r11 +; SSE2-NEXT: movd %r11d, %xmm4 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSE2-NEXT: shlq $59, %r14 +; SSE2-NEXT: sarq $63, %r14 +; SSE2-NEXT: movd %r14d, %xmm5 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] +; SSE2-NEXT: shlq $55, %r15 +; SSE2-NEXT: sarq $63, %r15 +; SSE2-NEXT: movd %r15d, %xmm2 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7] +; SSE2-NEXT: shlq $63, %r12 +; SSE2-NEXT: sarq $63, %r12 +; SSE2-NEXT: movd %r12d, %xmm0 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7] +; SSE2-NEXT: shlq $50, %r13 +; SSE2-NEXT: sarq $63, %r13 +; SSE2-NEXT: movd %r13d, %xmm1 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; SSE2-NEXT: shlq $58, %rbx +; SSE2-NEXT: sarq $63, %rbx +; SSE2-NEXT: movd %ebx, %xmm2 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7] +; SSE2-NEXT: shlq $54, %rcx +; SSE2-NEXT: sarq $63, %rcx +; SSE2-NEXT: movd %ecx, %xmm4 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] +; SSE2-NEXT: shlq $62, %rdx +; SSE2-NEXT: sarq $63, %rdx +; SSE2-NEXT: movd %edx, %xmm3 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] +; SSE2-NEXT: shlq $52, %rsi +; SSE2-NEXT: sarq $63, %rsi +; SSE2-NEXT: movd %esi, %xmm1 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] +; SSE2-NEXT: shlq $60, %rdi +; SSE2-NEXT: sarq $63, %rdi +; SSE2-NEXT: movd %edi, %xmm4 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3],xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7] +; SSE2-NEXT: shrq $15, %rbp +; SSE2-NEXT: movd %ebp, %xmm1 +; SSE2-NEXT: shrq $7, %rax +; SSE2-NEXT: movd %eax, %xmm2 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] +; SSE2-NEXT: popq %rbx +; SSE2-NEXT: popq %r12 +; SSE2-NEXT: popq %r13 +; SSE2-NEXT: popq %r14 +; SSE2-NEXT: popq %r15 +; SSE2-NEXT: popq %rbp +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_sext_16i1_to_16i8: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: pushq %rbp +; SSSE3-NEXT: pushq %r15 +; SSSE3-NEXT: pushq %r14 +; SSSE3-NEXT: pushq %r13 +; SSSE3-NEXT: pushq %r12 +; SSSE3-NEXT: pushq %rbx +; SSSE3-NEXT: movswq (%rdi), %rax +; SSSE3-NEXT: movq %rax, %r8 +; SSSE3-NEXT: movq %rax, %r9 +; SSSE3-NEXT: movq %rax, %r10 +; SSSE3-NEXT: movq %rax, %r11 +; SSSE3-NEXT: movq %rax, %r14 +; SSSE3-NEXT: movq %rax, %r15 +; SSSE3-NEXT: movq %rax, %r12 +; SSSE3-NEXT: movq %rax, %r13 +; SSSE3-NEXT: movq %rax, %rbx +; SSSE3-NEXT: movq %rax, %rcx +; SSSE3-NEXT: movq %rax, %rdx +; SSSE3-NEXT: movq %rax, %rsi +; SSSE3-NEXT: movq %rax, %rdi +; SSSE3-NEXT: movq %rax, %rbp +; SSSE3-NEXT: shlq $49, %rbp +; SSSE3-NEXT: sarq $63, %rbp +; SSSE3-NEXT: movd %ebp, %xmm0 +; SSSE3-NEXT: movq %rax, %rbp +; SSSE3-NEXT: movsbq %al, %rax +; SSSE3-NEXT: shlq $57, %r8 +; SSSE3-NEXT: sarq $63, %r8 +; SSSE3-NEXT: movd %r8d, %xmm1 +; SSSE3-NEXT: shlq $53, %r9 +; SSSE3-NEXT: sarq $63, %r9 +; SSSE3-NEXT: movd %r9d, %xmm2 +; SSSE3-NEXT: shlq $61, %r10 +; SSSE3-NEXT: sarq $63, %r10 +; SSSE3-NEXT: movd %r10d, %xmm3 +; SSSE3-NEXT: shlq $51, %r11 +; SSSE3-NEXT: sarq $63, %r11 +; SSSE3-NEXT: movd %r11d, %xmm4 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSSE3-NEXT: shlq $59, %r14 +; SSSE3-NEXT: sarq $63, %r14 +; SSSE3-NEXT: movd %r14d, %xmm5 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] +; SSSE3-NEXT: shlq $55, %r15 +; SSSE3-NEXT: sarq $63, %r15 +; SSSE3-NEXT: movd %r15d, %xmm2 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7] +; SSSE3-NEXT: shlq $63, %r12 +; SSSE3-NEXT: sarq $63, %r12 +; SSSE3-NEXT: movd %r12d, %xmm0 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7] +; SSSE3-NEXT: shlq $50, %r13 +; SSSE3-NEXT: sarq $63, %r13 +; SSSE3-NEXT: movd %r13d, %xmm1 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; SSSE3-NEXT: shlq $58, %rbx +; SSSE3-NEXT: sarq $63, %rbx +; SSSE3-NEXT: movd %ebx, %xmm2 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7] +; SSSE3-NEXT: shlq $54, %rcx +; SSSE3-NEXT: sarq $63, %rcx +; SSSE3-NEXT: movd %ecx, %xmm4 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] +; SSSE3-NEXT: shlq $62, %rdx +; SSSE3-NEXT: sarq $63, %rdx +; SSSE3-NEXT: movd %edx, %xmm3 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] +; SSSE3-NEXT: shlq $52, %rsi +; SSSE3-NEXT: sarq $63, %rsi +; SSSE3-NEXT: movd %esi, %xmm1 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] +; SSSE3-NEXT: shlq $60, %rdi +; SSSE3-NEXT: sarq $63, %rdi +; SSSE3-NEXT: movd %edi, %xmm4 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3],xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7] +; SSSE3-NEXT: shrq $15, %rbp +; SSSE3-NEXT: movd %ebp, %xmm1 +; SSSE3-NEXT: shrq $7, %rax +; SSSE3-NEXT: movd %eax, %xmm2 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] +; SSSE3-NEXT: popq %rbx +; SSSE3-NEXT: popq %r12 +; SSSE3-NEXT: popq %r13 +; SSSE3-NEXT: popq %r14 +; SSSE3-NEXT: popq %r15 +; SSSE3-NEXT: popq %rbp +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_sext_16i1_to_16i8: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: movswq (%rdi), %rax +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $62, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: movq %rax, %rdx +; SSE41-NEXT: shlq $63, %rdx +; SSE41-NEXT: sarq $63, %rdx +; SSE41-NEXT: movd %edx, %xmm0 +; SSE41-NEXT: pinsrb $1, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $61, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $2, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $60, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $3, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $59, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $4, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $58, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $5, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $57, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $6, %ecx, %xmm0 +; SSE41-NEXT: movsbq %al, %rcx +; SSE41-NEXT: shrq $7, %rcx +; SSE41-NEXT: pinsrb $7, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $55, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $8, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $54, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $9, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $53, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $10, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $52, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $11, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $51, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $12, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $50, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $13, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $49, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $14, %ecx, %xmm0 +; SSE41-NEXT: shrq $15, %rax +; SSE41-NEXT: pinsrb $15, %eax, %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: load_sext_16i1_to_16i8: +; AVX: # BB#0: # %entry +; AVX-NEXT: movswq (%rdi), %rax +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $62, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: movq %rax, %rdx +; AVX-NEXT: shlq $63, %rdx +; AVX-NEXT: sarq $63, %rdx +; AVX-NEXT: vmovd %edx, %xmm0 +; AVX-NEXT: vpinsrb $1, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $61, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrb $2, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $60, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrb $3, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $59, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrb $4, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $58, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrb $5, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $57, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrb $6, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movsbq %al, %rcx +; AVX-NEXT: shrq $7, %rcx +; AVX-NEXT: vpinsrb $7, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $55, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrb $8, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $54, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrb $9, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $53, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrb $10, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $52, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrb $11, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $51, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrb $12, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $50, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrb $13, %ecx, %xmm0, %xmm0 +; AVX-NEXT: movq %rax, %rcx +; AVX-NEXT: shlq $49, %rcx +; AVX-NEXT: sarq $63, %rcx +; AVX-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0 +; AVX-NEXT: shrq $15, %rax +; AVX-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_16i1_to_16i8: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-SSE41-NEXT: movswl (%eax), %eax +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $30, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: movl %eax, %edx +; X32-SSE41-NEXT: shll $31, %edx +; X32-SSE41-NEXT: sarl $31, %edx +; X32-SSE41-NEXT: movd %edx, %xmm0 +; X32-SSE41-NEXT: pinsrb $1, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $29, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $2, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $28, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $3, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $27, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $4, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $26, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $5, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $25, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $6, %ecx, %xmm0 +; X32-SSE41-NEXT: movsbl %al, %ecx +; X32-SSE41-NEXT: shrl $7, %ecx +; X32-SSE41-NEXT: pinsrb $7, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $23, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $8, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $22, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $9, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $21, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $10, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $20, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $11, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $19, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $12, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $18, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $13, %ecx, %xmm0 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $17, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $14, %ecx, %xmm0 +; X32-SSE41-NEXT: shrl $15, %eax +; X32-SSE41-NEXT: pinsrb $15, %eax, %xmm0 +; X32-SSE41-NEXT: retl +entry: + %X = load <16 x i1>, <16 x i1>* %ptr + %Y = sext <16 x i1> %X to <16 x i8> + ret <16 x i8> %Y +} + +define <16 x i16> @load_sext_16i1_to_16i16(<16 x i1> *%ptr) { +; SSE2-LABEL: load_sext_16i1_to_16i16: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movzwl (%rdi), %eax +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $14, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm0 +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $6, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm1 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $10, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm0 +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $2, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm2 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $12, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm0 +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $4, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm3 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm1 +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $8, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm0 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $13, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm0 +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $5, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm2 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $9, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm3 +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm0 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $11, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm2 +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $3, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm3 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] +; SSE2-NEXT: movl %eax, %ecx +; SSE2-NEXT: shrl $7, %ecx +; SSE2-NEXT: andl $1, %ecx +; SSE2-NEXT: movd %ecx, %xmm2 +; SSE2-NEXT: shrl $15, %eax +; SSE2-NEXT: movzwl %ax, %eax +; SSE2-NEXT: movd %eax, %xmm4 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: psllw $15, %xmm0 +; SSE2-NEXT: psraw $15, %xmm0 +; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] +; SSE2-NEXT: psllw $15, %xmm1 +; SSE2-NEXT: psraw $15, %xmm1 +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_sext_16i1_to_16i16: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movzwl (%rdi), %eax +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $14, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm0 +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $6, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm1 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $10, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm0 +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $2, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm2 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $12, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm0 +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $4, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm3 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm1 +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $8, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm0 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $13, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm0 +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $5, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm2 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $9, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm3 +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm0 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $11, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm2 +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $3, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm3 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] +; SSSE3-NEXT: movl %eax, %ecx +; SSSE3-NEXT: shrl $7, %ecx +; SSSE3-NEXT: andl $1, %ecx +; SSSE3-NEXT: movd %ecx, %xmm2 +; SSSE3-NEXT: shrl $15, %eax +; SSSE3-NEXT: movzwl %ax, %eax +; SSSE3-NEXT: movd %eax, %xmm4 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSSE3-NEXT: psllw $15, %xmm0 +; SSSE3-NEXT: psraw $15, %xmm0 +; SSSE3-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] +; SSSE3-NEXT: psllw $15, %xmm1 +; SSSE3-NEXT: psraw $15, %xmm1 +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_sext_16i1_to_16i16: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: movzwl (%rdi), %eax +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: movl %eax, %edx +; SSE41-NEXT: andl $1, %edx +; SSE41-NEXT: movd %edx, %xmm1 +; SSE41-NEXT: pinsrb $1, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $2, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrb $2, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $3, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrb $3, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $4, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrb $4, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $5, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrb $5, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $6, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrb $6, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $7, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrb $7, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $8, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrb $8, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $9, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrb $9, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $10, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrb $10, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $11, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrb $11, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $12, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrb $12, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $13, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrb $13, %ecx, %xmm1 +; SSE41-NEXT: movl %eax, %ecx +; SSE41-NEXT: shrl $14, %ecx +; SSE41-NEXT: andl $1, %ecx +; SSE41-NEXT: pinsrb $14, %ecx, %xmm1 +; SSE41-NEXT: shrl $15, %eax +; SSE41-NEXT: movzwl %ax, %eax +; SSE41-NEXT: pinsrb $15, %eax, %xmm1 +; SSE41-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero +; SSE41-NEXT: psllw $15, %xmm0 +; SSE41-NEXT: psraw $15, %xmm0 +; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] +; SSE41-NEXT: psllw $15, %xmm1 +; SSE41-NEXT: psraw $15, %xmm1 +; SSE41-NEXT: retq +; +; AVX1-LABEL: load_sext_16i1_to_16i16: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: pushq %rbp +; AVX1-NEXT: .Ltmp0: +; AVX1-NEXT: .cfi_def_cfa_offset 16 +; AVX1-NEXT: pushq %r15 +; AVX1-NEXT: .Ltmp1: +; AVX1-NEXT: .cfi_def_cfa_offset 24 +; AVX1-NEXT: pushq %r14 +; AVX1-NEXT: .Ltmp2: +; AVX1-NEXT: .cfi_def_cfa_offset 32 +; AVX1-NEXT: pushq %r13 +; AVX1-NEXT: .Ltmp3: +; AVX1-NEXT: .cfi_def_cfa_offset 40 +; AVX1-NEXT: pushq %r12 +; AVX1-NEXT: .Ltmp4: +; AVX1-NEXT: .cfi_def_cfa_offset 48 +; AVX1-NEXT: pushq %rbx +; AVX1-NEXT: .Ltmp5: +; AVX1-NEXT: .cfi_def_cfa_offset 56 +; AVX1-NEXT: .Ltmp6: +; AVX1-NEXT: .cfi_offset %rbx, -56 +; AVX1-NEXT: .Ltmp7: +; AVX1-NEXT: .cfi_offset %r12, -48 +; AVX1-NEXT: .Ltmp8: +; AVX1-NEXT: .cfi_offset %r13, -40 +; AVX1-NEXT: .Ltmp9: +; AVX1-NEXT: .cfi_offset %r14, -32 +; AVX1-NEXT: .Ltmp10: +; AVX1-NEXT: .cfi_offset %r15, -24 +; AVX1-NEXT: .Ltmp11: +; AVX1-NEXT: .cfi_offset %rbp, -16 +; AVX1-NEXT: movswq (%rdi), %rax +; AVX1-NEXT: movq %rax, %rcx +; AVX1-NEXT: shlq $55, %rcx +; AVX1-NEXT: sarq $63, %rcx +; AVX1-NEXT: vmovd %ecx, %xmm0 +; AVX1-NEXT: movq %rax, %r8 +; AVX1-NEXT: movq %rax, %r10 +; AVX1-NEXT: movq %rax, %r11 +; AVX1-NEXT: movq %rax, %r14 +; AVX1-NEXT: movq %rax, %r15 +; AVX1-NEXT: movq %rax, %r9 +; AVX1-NEXT: movq %rax, %r12 +; AVX1-NEXT: movq %rax, %r13 +; AVX1-NEXT: movq %rax, %rbx +; AVX1-NEXT: movq %rax, %rdi +; AVX1-NEXT: movq %rax, %rcx +; AVX1-NEXT: movq %rax, %rdx +; AVX1-NEXT: movq %rax, %rsi +; AVX1-NEXT: movsbq %al, %rbp +; AVX1-NEXT: shlq $54, %rax +; AVX1-NEXT: sarq $63, %rax +; AVX1-NEXT: vpinsrw $1, %eax, %xmm0, %xmm0 +; AVX1-NEXT: shlq $53, %r8 +; AVX1-NEXT: sarq $63, %r8 +; AVX1-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0 +; AVX1-NEXT: shlq $52, %r10 +; AVX1-NEXT: sarq $63, %r10 +; AVX1-NEXT: vpinsrw $3, %r10d, %xmm0, %xmm0 +; AVX1-NEXT: shlq $51, %r11 +; AVX1-NEXT: sarq $63, %r11 +; AVX1-NEXT: vpinsrw $4, %r11d, %xmm0, %xmm0 +; AVX1-NEXT: shlq $50, %r14 +; AVX1-NEXT: sarq $63, %r14 +; AVX1-NEXT: vpinsrw $5, %r14d, %xmm0, %xmm0 +; AVX1-NEXT: shlq $49, %r15 +; AVX1-NEXT: sarq $63, %r15 +; AVX1-NEXT: vpinsrw $6, %r15d, %xmm0, %xmm0 +; AVX1-NEXT: shrq $15, %r9 +; AVX1-NEXT: vpinsrw $7, %r9d, %xmm0, %xmm0 +; AVX1-NEXT: shlq $63, %r13 +; AVX1-NEXT: sarq $63, %r13 +; AVX1-NEXT: vmovd %r13d, %xmm1 +; AVX1-NEXT: shlq $62, %r12 +; AVX1-NEXT: sarq $63, %r12 +; AVX1-NEXT: vpinsrw $1, %r12d, %xmm1, %xmm1 +; AVX1-NEXT: shlq $61, %rbx +; AVX1-NEXT: sarq $63, %rbx +; AVX1-NEXT: vpinsrw $2, %ebx, %xmm1, %xmm1 +; AVX1-NEXT: shlq $60, %rdi +; AVX1-NEXT: sarq $63, %rdi +; AVX1-NEXT: vpinsrw $3, %edi, %xmm1, %xmm1 +; AVX1-NEXT: shlq $59, %rcx +; AVX1-NEXT: sarq $63, %rcx +; AVX1-NEXT: vpinsrw $4, %ecx, %xmm1, %xmm1 +; AVX1-NEXT: shlq $58, %rdx +; AVX1-NEXT: sarq $63, %rdx +; AVX1-NEXT: vpinsrw $5, %edx, %xmm1, %xmm1 +; AVX1-NEXT: shlq $57, %rsi +; AVX1-NEXT: sarq $63, %rsi +; AVX1-NEXT: vpinsrw $6, %esi, %xmm1, %xmm1 +; AVX1-NEXT: shrq $7, %rbp +; AVX1-NEXT: vpinsrw $7, %ebp, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: popq %rbx +; AVX1-NEXT: popq %r12 +; AVX1-NEXT: popq %r13 +; AVX1-NEXT: popq %r14 +; AVX1-NEXT: popq %r15 +; AVX1-NEXT: popq %rbp +; AVX1-NEXT: retq +; +; AVX2-LABEL: load_sext_16i1_to_16i16: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: pushq %rbp +; AVX2-NEXT: .Ltmp0: +; AVX2-NEXT: .cfi_def_cfa_offset 16 +; AVX2-NEXT: pushq %r15 +; AVX2-NEXT: .Ltmp1: +; AVX2-NEXT: .cfi_def_cfa_offset 24 +; AVX2-NEXT: pushq %r14 +; AVX2-NEXT: .Ltmp2: +; AVX2-NEXT: .cfi_def_cfa_offset 32 +; AVX2-NEXT: pushq %r13 +; AVX2-NEXT: .Ltmp3: +; AVX2-NEXT: .cfi_def_cfa_offset 40 +; AVX2-NEXT: pushq %r12 +; AVX2-NEXT: .Ltmp4: +; AVX2-NEXT: .cfi_def_cfa_offset 48 +; AVX2-NEXT: pushq %rbx +; AVX2-NEXT: .Ltmp5: +; AVX2-NEXT: .cfi_def_cfa_offset 56 +; AVX2-NEXT: .Ltmp6: +; AVX2-NEXT: .cfi_offset %rbx, -56 +; AVX2-NEXT: .Ltmp7: +; AVX2-NEXT: .cfi_offset %r12, -48 +; AVX2-NEXT: .Ltmp8: +; AVX2-NEXT: .cfi_offset %r13, -40 +; AVX2-NEXT: .Ltmp9: +; AVX2-NEXT: .cfi_offset %r14, -32 +; AVX2-NEXT: .Ltmp10: +; AVX2-NEXT: .cfi_offset %r15, -24 +; AVX2-NEXT: .Ltmp11: +; AVX2-NEXT: .cfi_offset %rbp, -16 +; AVX2-NEXT: movswq (%rdi), %rax +; AVX2-NEXT: movq %rax, %rcx +; AVX2-NEXT: shlq $55, %rcx +; AVX2-NEXT: sarq $63, %rcx +; AVX2-NEXT: vmovd %ecx, %xmm0 +; AVX2-NEXT: movq %rax, %r8 +; AVX2-NEXT: movq %rax, %r10 +; AVX2-NEXT: movq %rax, %r11 +; AVX2-NEXT: movq %rax, %r14 +; AVX2-NEXT: movq %rax, %r15 +; AVX2-NEXT: movq %rax, %r9 +; AVX2-NEXT: movq %rax, %r12 +; AVX2-NEXT: movq %rax, %r13 +; AVX2-NEXT: movq %rax, %rbx +; AVX2-NEXT: movq %rax, %rdi +; AVX2-NEXT: movq %rax, %rcx +; AVX2-NEXT: movq %rax, %rdx +; AVX2-NEXT: movq %rax, %rsi +; AVX2-NEXT: movsbq %al, %rbp +; AVX2-NEXT: shlq $54, %rax +; AVX2-NEXT: sarq $63, %rax +; AVX2-NEXT: vpinsrw $1, %eax, %xmm0, %xmm0 +; AVX2-NEXT: shlq $53, %r8 +; AVX2-NEXT: sarq $63, %r8 +; AVX2-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0 +; AVX2-NEXT: shlq $52, %r10 +; AVX2-NEXT: sarq $63, %r10 +; AVX2-NEXT: vpinsrw $3, %r10d, %xmm0, %xmm0 +; AVX2-NEXT: shlq $51, %r11 +; AVX2-NEXT: sarq $63, %r11 +; AVX2-NEXT: vpinsrw $4, %r11d, %xmm0, %xmm0 +; AVX2-NEXT: shlq $50, %r14 +; AVX2-NEXT: sarq $63, %r14 +; AVX2-NEXT: vpinsrw $5, %r14d, %xmm0, %xmm0 +; AVX2-NEXT: shlq $49, %r15 +; AVX2-NEXT: sarq $63, %r15 +; AVX2-NEXT: vpinsrw $6, %r15d, %xmm0, %xmm0 +; AVX2-NEXT: shrq $15, %r9 +; AVX2-NEXT: vpinsrw $7, %r9d, %xmm0, %xmm0 +; AVX2-NEXT: shlq $63, %r13 +; AVX2-NEXT: sarq $63, %r13 +; AVX2-NEXT: vmovd %r13d, %xmm1 +; AVX2-NEXT: shlq $62, %r12 +; AVX2-NEXT: sarq $63, %r12 +; AVX2-NEXT: vpinsrw $1, %r12d, %xmm1, %xmm1 +; AVX2-NEXT: shlq $61, %rbx +; AVX2-NEXT: sarq $63, %rbx +; AVX2-NEXT: vpinsrw $2, %ebx, %xmm1, %xmm1 +; AVX2-NEXT: shlq $60, %rdi +; AVX2-NEXT: sarq $63, %rdi +; AVX2-NEXT: vpinsrw $3, %edi, %xmm1, %xmm1 +; AVX2-NEXT: shlq $59, %rcx +; AVX2-NEXT: sarq $63, %rcx +; AVX2-NEXT: vpinsrw $4, %ecx, %xmm1, %xmm1 +; AVX2-NEXT: shlq $58, %rdx +; AVX2-NEXT: sarq $63, %rdx +; AVX2-NEXT: vpinsrw $5, %edx, %xmm1, %xmm1 +; AVX2-NEXT: shlq $57, %rsi +; AVX2-NEXT: sarq $63, %rsi +; AVX2-NEXT: vpinsrw $6, %esi, %xmm1, %xmm1 +; AVX2-NEXT: shrq $7, %rbp +; AVX2-NEXT: vpinsrw $7, %ebp, %xmm1, %xmm1 +; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX2-NEXT: popq %rbx +; AVX2-NEXT: popq %r12 +; AVX2-NEXT: popq %r13 +; AVX2-NEXT: popq %r14 +; AVX2-NEXT: popq %r15 +; AVX2-NEXT: popq %rbp +; AVX2-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_16i1_to_16i16: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-SSE41-NEXT: movzwl (%eax), %eax +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: movl %eax, %edx +; X32-SSE41-NEXT: andl $1, %edx +; X32-SSE41-NEXT: movd %edx, %xmm1 +; X32-SSE41-NEXT: pinsrb $1, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $2, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrb $2, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $3, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrb $3, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $4, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrb $4, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $5, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrb $5, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $6, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrb $6, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $7, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrb $7, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $8, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrb $8, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $9, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrb $9, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $10, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrb $10, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $11, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrb $11, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $12, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrb $12, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $13, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrb $13, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shrl $14, %ecx +; X32-SSE41-NEXT: andl $1, %ecx +; X32-SSE41-NEXT: pinsrb $14, %ecx, %xmm1 +; X32-SSE41-NEXT: shrl $15, %eax +; X32-SSE41-NEXT: pinsrb $15, %eax, %xmm1 +; X32-SSE41-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero +; X32-SSE41-NEXT: psllw $15, %xmm0 +; X32-SSE41-NEXT: psraw $15, %xmm0 +; X32-SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] +; X32-SSE41-NEXT: psllw $15, %xmm1 +; X32-SSE41-NEXT: psraw $15, %xmm1 +; X32-SSE41-NEXT: retl +entry: + %X = load <16 x i1>, <16 x i1>* %ptr + %Y = sext <16 x i1> %X to <16 x i16> + ret <16 x i16> %Y +} + +define <32 x i8> @load_sext_32i1_to_32i8(<32 x i1> *%ptr) nounwind readnone { +; SSE2-LABEL: load_sext_32i1_to_32i8: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: pushq %rbp +; SSE2-NEXT: pushq %r15 +; SSE2-NEXT: pushq %r14 +; SSE2-NEXT: pushq %r13 +; SSE2-NEXT: pushq %r12 +; SSE2-NEXT: pushq %rbx +; SSE2-NEXT: movswq (%rdi), %rbx +; SSE2-NEXT: movq %rbx, %r10 +; SSE2-NEXT: movq %rbx, %r8 +; SSE2-NEXT: movq %rbx, %r9 +; SSE2-NEXT: movq %rbx, %r11 +; SSE2-NEXT: movq %rbx, %r14 +; SSE2-NEXT: movq %rbx, %r15 +; SSE2-NEXT: movq %rbx, %r12 +; SSE2-NEXT: movq %rbx, %r13 +; SSE2-NEXT: movq %rbx, %rdx +; SSE2-NEXT: movq %rbx, %rsi +; SSE2-NEXT: movq %rbx, %rcx +; SSE2-NEXT: movq %rbx, %rbp +; SSE2-NEXT: movq %rbx, %rax +; SSE2-NEXT: shlq $49, %rax +; SSE2-NEXT: sarq $63, %rax +; SSE2-NEXT: movd %eax, %xmm0 +; SSE2-NEXT: movq %rbx, %rax +; SSE2-NEXT: shlq $57, %r10 +; SSE2-NEXT: sarq $63, %r10 +; SSE2-NEXT: movd %r10d, %xmm15 +; SSE2-NEXT: movq %rbx, %r10 +; SSE2-NEXT: movsbq %bl, %rbx +; SSE2-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3],xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7] +; SSE2-NEXT: shlq $53, %r8 +; SSE2-NEXT: sarq $63, %r8 +; SSE2-NEXT: movd %r8d, %xmm8 +; SSE2-NEXT: shlq $61, %r9 +; SSE2-NEXT: sarq $63, %r9 +; SSE2-NEXT: movd %r9d, %xmm2 +; SSE2-NEXT: shlq $51, %r11 +; SSE2-NEXT: sarq $63, %r11 +; SSE2-NEXT: movd %r11d, %xmm9 +; SSE2-NEXT: shlq $59, %r14 +; SSE2-NEXT: sarq $63, %r14 +; SSE2-NEXT: movd %r14d, %xmm5 +; SSE2-NEXT: shlq $55, %r15 +; SSE2-NEXT: sarq $63, %r15 +; SSE2-NEXT: movd %r15d, %xmm10 +; SSE2-NEXT: shlq $63, %r12 +; SSE2-NEXT: sarq $63, %r12 +; SSE2-NEXT: movd %r12d, %xmm0 +; SSE2-NEXT: shlq $50, %r13 +; SSE2-NEXT: sarq $63, %r13 +; SSE2-NEXT: movd %r13d, %xmm11 +; SSE2-NEXT: shlq $58, %rdx +; SSE2-NEXT: sarq $63, %rdx +; SSE2-NEXT: movd %edx, %xmm4 +; SSE2-NEXT: shlq $54, %rsi +; SSE2-NEXT: sarq $63, %rsi +; SSE2-NEXT: movd %esi, %xmm12 +; SSE2-NEXT: shlq $62, %rcx +; SSE2-NEXT: sarq $63, %rcx +; SSE2-NEXT: movd %ecx, %xmm6 +; SSE2-NEXT: shlq $52, %rbp +; SSE2-NEXT: sarq $63, %rbp +; SSE2-NEXT: movd %ebp, %xmm13 +; SSE2-NEXT: shlq $60, %rax +; SSE2-NEXT: sarq $63, %rax +; SSE2-NEXT: movd %eax, %xmm7 +; SSE2-NEXT: shrq $15, %r10 +; SSE2-NEXT: movd %r10d, %xmm14 +; SSE2-NEXT: shrq $7, %rbx +; SSE2-NEXT: movd %ebx, %xmm3 +; SSE2-NEXT: movswq 2(%rdi), %rdx +; SSE2-NEXT: movq %rdx, %r8 +; SSE2-NEXT: movq %rdx, %r9 +; SSE2-NEXT: movq %rdx, %r10 +; SSE2-NEXT: movq %rdx, %r11 +; SSE2-NEXT: movq %rdx, %r14 +; SSE2-NEXT: movq %rdx, %r15 +; SSE2-NEXT: movq %rdx, %r12 +; SSE2-NEXT: movq %rdx, %r13 +; SSE2-NEXT: movq %rdx, %rbx +; SSE2-NEXT: movq %rdx, %rax +; SSE2-NEXT: movq %rdx, %rcx +; SSE2-NEXT: movq %rdx, %rsi +; SSE2-NEXT: movq %rdx, %rdi +; SSE2-NEXT: movq %rdx, %rbp +; SSE2-NEXT: shlq $49, %rbp +; SSE2-NEXT: sarq $63, %rbp +; SSE2-NEXT: movd %ebp, %xmm1 +; SSE2-NEXT: movq %rdx, %rbp +; SSE2-NEXT: movsbq %dl, %rdx +; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm8[0],xmm2[1],xmm8[1],xmm2[2],xmm8[2],xmm2[3],xmm8[3],xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3],xmm2[4],xmm15[4],xmm2[5],xmm15[5],xmm2[6],xmm15[6],xmm2[7],xmm15[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3],xmm5[4],xmm9[4],xmm5[5],xmm9[5],xmm5[6],xmm9[6],xmm5[7],xmm9[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm11[0],xmm4[1],xmm11[1],xmm4[2],xmm11[2],xmm4[3],xmm11[3],xmm4[4],xmm11[4],xmm4[5],xmm11[5],xmm4[6],xmm11[6],xmm4[7],xmm11[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm12[0],xmm6[1],xmm12[1],xmm6[2],xmm12[2],xmm6[3],xmm12[3],xmm6[4],xmm12[4],xmm6[5],xmm12[5],xmm6[6],xmm12[6],xmm6[7],xmm12[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm13[0],xmm7[1],xmm13[1],xmm7[2],xmm13[2],xmm7[3],xmm13[3],xmm7[4],xmm13[4],xmm7[5],xmm13[5],xmm7[6],xmm13[6],xmm7[7],xmm13[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm14[0],xmm3[1],xmm14[1],xmm3[2],xmm14[2],xmm3[3],xmm14[3],xmm3[4],xmm14[4],xmm3[5],xmm14[5],xmm3[6],xmm14[6],xmm3[7],xmm14[7] +; SSE2-NEXT: shlq $57, %r8 +; SSE2-NEXT: sarq $63, %r8 +; SSE2-NEXT: movd %r8d, %xmm2 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7] +; SSE2-NEXT: shlq $53, %r9 +; SSE2-NEXT: sarq $63, %r9 +; SSE2-NEXT: movd %r9d, %xmm3 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3],xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7] +; SSE2-NEXT: shlq $61, %r10 +; SSE2-NEXT: sarq $63, %r10 +; SSE2-NEXT: movd %r10d, %xmm4 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3],xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7] +; SSE2-NEXT: shlq $51, %r11 +; SSE2-NEXT: sarq $63, %r11 +; SSE2-NEXT: movd %r11d, %xmm5 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] +; SSE2-NEXT: shlq $59, %r14 +; SSE2-NEXT: sarq $63, %r14 +; SSE2-NEXT: movd %r14d, %xmm6 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7] +; SSE2-NEXT: shlq $55, %r15 +; SSE2-NEXT: sarq $63, %r15 +; SSE2-NEXT: movd %r15d, %xmm3 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7] +; SSE2-NEXT: shlq $63, %r12 +; SSE2-NEXT: sarq $63, %r12 +; SSE2-NEXT: movd %r12d, %xmm1 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7] +; SSE2-NEXT: shlq $50, %r13 +; SSE2-NEXT: sarq $63, %r13 +; SSE2-NEXT: movd %r13d, %xmm2 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7] +; SSE2-NEXT: shlq $58, %rbx +; SSE2-NEXT: sarq $63, %rbx +; SSE2-NEXT: movd %ebx, %xmm3 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7] +; SSE2-NEXT: shlq $54, %rax +; SSE2-NEXT: sarq $63, %rax +; SSE2-NEXT: movd %eax, %xmm5 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; SSE2-NEXT: shlq $62, %rcx +; SSE2-NEXT: sarq $63, %rcx +; SSE2-NEXT: movd %ecx, %xmm4 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] +; SSE2-NEXT: shlq $52, %rsi +; SSE2-NEXT: sarq $63, %rsi +; SSE2-NEXT: movd %esi, %xmm2 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7] +; SSE2-NEXT: shlq $60, %rdi +; SSE2-NEXT: sarq $63, %rdi +; SSE2-NEXT: movd %edi, %xmm3 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] +; SSE2-NEXT: shrq $15, %rbp +; SSE2-NEXT: movd %ebp, %xmm2 +; SSE2-NEXT: shrq $7, %rdx +; SSE2-NEXT: movd %edx, %xmm5 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7] +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; SSE2-NEXT: popq %rbx +; SSE2-NEXT: popq %r12 +; SSE2-NEXT: popq %r13 +; SSE2-NEXT: popq %r14 +; SSE2-NEXT: popq %r15 +; SSE2-NEXT: popq %rbp +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_sext_32i1_to_32i8: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: pushq %rbp +; SSSE3-NEXT: pushq %r15 +; SSSE3-NEXT: pushq %r14 +; SSSE3-NEXT: pushq %r13 +; SSSE3-NEXT: pushq %r12 +; SSSE3-NEXT: pushq %rbx +; SSSE3-NEXT: movswq (%rdi), %rbx +; SSSE3-NEXT: movq %rbx, %r10 +; SSSE3-NEXT: movq %rbx, %r8 +; SSSE3-NEXT: movq %rbx, %r9 +; SSSE3-NEXT: movq %rbx, %r11 +; SSSE3-NEXT: movq %rbx, %r14 +; SSSE3-NEXT: movq %rbx, %r15 +; SSSE3-NEXT: movq %rbx, %r12 +; SSSE3-NEXT: movq %rbx, %r13 +; SSSE3-NEXT: movq %rbx, %rdx +; SSSE3-NEXT: movq %rbx, %rsi +; SSSE3-NEXT: movq %rbx, %rcx +; SSSE3-NEXT: movq %rbx, %rbp +; SSSE3-NEXT: movq %rbx, %rax +; SSSE3-NEXT: shlq $49, %rax +; SSSE3-NEXT: sarq $63, %rax +; SSSE3-NEXT: movd %eax, %xmm0 +; SSSE3-NEXT: movq %rbx, %rax +; SSSE3-NEXT: shlq $57, %r10 +; SSSE3-NEXT: sarq $63, %r10 +; SSSE3-NEXT: movd %r10d, %xmm15 +; SSSE3-NEXT: movq %rbx, %r10 +; SSSE3-NEXT: movsbq %bl, %rbx +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3],xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7] +; SSSE3-NEXT: shlq $53, %r8 +; SSSE3-NEXT: sarq $63, %r8 +; SSSE3-NEXT: movd %r8d, %xmm8 +; SSSE3-NEXT: shlq $61, %r9 +; SSSE3-NEXT: sarq $63, %r9 +; SSSE3-NEXT: movd %r9d, %xmm2 +; SSSE3-NEXT: shlq $51, %r11 +; SSSE3-NEXT: sarq $63, %r11 +; SSSE3-NEXT: movd %r11d, %xmm9 +; SSSE3-NEXT: shlq $59, %r14 +; SSSE3-NEXT: sarq $63, %r14 +; SSSE3-NEXT: movd %r14d, %xmm5 +; SSSE3-NEXT: shlq $55, %r15 +; SSSE3-NEXT: sarq $63, %r15 +; SSSE3-NEXT: movd %r15d, %xmm10 +; SSSE3-NEXT: shlq $63, %r12 +; SSSE3-NEXT: sarq $63, %r12 +; SSSE3-NEXT: movd %r12d, %xmm0 +; SSSE3-NEXT: shlq $50, %r13 +; SSSE3-NEXT: sarq $63, %r13 +; SSSE3-NEXT: movd %r13d, %xmm11 +; SSSE3-NEXT: shlq $58, %rdx +; SSSE3-NEXT: sarq $63, %rdx +; SSSE3-NEXT: movd %edx, %xmm4 +; SSSE3-NEXT: shlq $54, %rsi +; SSSE3-NEXT: sarq $63, %rsi +; SSSE3-NEXT: movd %esi, %xmm12 +; SSSE3-NEXT: shlq $62, %rcx +; SSSE3-NEXT: sarq $63, %rcx +; SSSE3-NEXT: movd %ecx, %xmm6 +; SSSE3-NEXT: shlq $52, %rbp +; SSSE3-NEXT: sarq $63, %rbp +; SSSE3-NEXT: movd %ebp, %xmm13 +; SSSE3-NEXT: shlq $60, %rax +; SSSE3-NEXT: sarq $63, %rax +; SSSE3-NEXT: movd %eax, %xmm7 +; SSSE3-NEXT: shrq $15, %r10 +; SSSE3-NEXT: movd %r10d, %xmm14 +; SSSE3-NEXT: shrq $7, %rbx +; SSSE3-NEXT: movd %ebx, %xmm3 +; SSSE3-NEXT: movswq 2(%rdi), %rdx +; SSSE3-NEXT: movq %rdx, %r8 +; SSSE3-NEXT: movq %rdx, %r9 +; SSSE3-NEXT: movq %rdx, %r10 +; SSSE3-NEXT: movq %rdx, %r11 +; SSSE3-NEXT: movq %rdx, %r14 +; SSSE3-NEXT: movq %rdx, %r15 +; SSSE3-NEXT: movq %rdx, %r12 +; SSSE3-NEXT: movq %rdx, %r13 +; SSSE3-NEXT: movq %rdx, %rbx +; SSSE3-NEXT: movq %rdx, %rax +; SSSE3-NEXT: movq %rdx, %rcx +; SSSE3-NEXT: movq %rdx, %rsi +; SSSE3-NEXT: movq %rdx, %rdi +; SSSE3-NEXT: movq %rdx, %rbp +; SSSE3-NEXT: shlq $49, %rbp +; SSSE3-NEXT: sarq $63, %rbp +; SSSE3-NEXT: movd %ebp, %xmm1 +; SSSE3-NEXT: movq %rdx, %rbp +; SSSE3-NEXT: movsbq %dl, %rdx +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm8[0],xmm2[1],xmm8[1],xmm2[2],xmm8[2],xmm2[3],xmm8[3],xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3],xmm2[4],xmm15[4],xmm2[5],xmm15[5],xmm2[6],xmm15[6],xmm2[7],xmm15[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3],xmm5[4],xmm9[4],xmm5[5],xmm9[5],xmm5[6],xmm9[6],xmm5[7],xmm9[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm11[0],xmm4[1],xmm11[1],xmm4[2],xmm11[2],xmm4[3],xmm11[3],xmm4[4],xmm11[4],xmm4[5],xmm11[5],xmm4[6],xmm11[6],xmm4[7],xmm11[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm12[0],xmm6[1],xmm12[1],xmm6[2],xmm12[2],xmm6[3],xmm12[3],xmm6[4],xmm12[4],xmm6[5],xmm12[5],xmm6[6],xmm12[6],xmm6[7],xmm12[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm13[0],xmm7[1],xmm13[1],xmm7[2],xmm13[2],xmm7[3],xmm13[3],xmm7[4],xmm13[4],xmm7[5],xmm13[5],xmm7[6],xmm13[6],xmm7[7],xmm13[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm14[0],xmm3[1],xmm14[1],xmm3[2],xmm14[2],xmm3[3],xmm14[3],xmm3[4],xmm14[4],xmm3[5],xmm14[5],xmm3[6],xmm14[6],xmm3[7],xmm14[7] +; SSSE3-NEXT: shlq $57, %r8 +; SSSE3-NEXT: sarq $63, %r8 +; SSSE3-NEXT: movd %r8d, %xmm2 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7] +; SSSE3-NEXT: shlq $53, %r9 +; SSSE3-NEXT: sarq $63, %r9 +; SSSE3-NEXT: movd %r9d, %xmm3 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3],xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7] +; SSSE3-NEXT: shlq $61, %r10 +; SSSE3-NEXT: sarq $63, %r10 +; SSSE3-NEXT: movd %r10d, %xmm4 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3],xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7] +; SSSE3-NEXT: shlq $51, %r11 +; SSSE3-NEXT: sarq $63, %r11 +; SSSE3-NEXT: movd %r11d, %xmm5 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] +; SSSE3-NEXT: shlq $59, %r14 +; SSSE3-NEXT: sarq $63, %r14 +; SSSE3-NEXT: movd %r14d, %xmm6 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7] +; SSSE3-NEXT: shlq $55, %r15 +; SSSE3-NEXT: sarq $63, %r15 +; SSSE3-NEXT: movd %r15d, %xmm3 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7] +; SSSE3-NEXT: shlq $63, %r12 +; SSSE3-NEXT: sarq $63, %r12 +; SSSE3-NEXT: movd %r12d, %xmm1 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7] +; SSSE3-NEXT: shlq $50, %r13 +; SSSE3-NEXT: sarq $63, %r13 +; SSSE3-NEXT: movd %r13d, %xmm2 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7] +; SSSE3-NEXT: shlq $58, %rbx +; SSSE3-NEXT: sarq $63, %rbx +; SSSE3-NEXT: movd %ebx, %xmm3 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7] +; SSSE3-NEXT: shlq $54, %rax +; SSSE3-NEXT: sarq $63, %rax +; SSSE3-NEXT: movd %eax, %xmm5 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; SSSE3-NEXT: shlq $62, %rcx +; SSSE3-NEXT: sarq $63, %rcx +; SSSE3-NEXT: movd %ecx, %xmm4 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] +; SSSE3-NEXT: shlq $52, %rsi +; SSSE3-NEXT: sarq $63, %rsi +; SSSE3-NEXT: movd %esi, %xmm2 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7] +; SSSE3-NEXT: shlq $60, %rdi +; SSSE3-NEXT: sarq $63, %rdi +; SSSE3-NEXT: movd %edi, %xmm3 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] +; SSSE3-NEXT: shrq $15, %rbp +; SSSE3-NEXT: movd %ebp, %xmm2 +; SSSE3-NEXT: shrq $7, %rdx +; SSSE3-NEXT: movd %edx, %xmm5 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7] +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; SSSE3-NEXT: popq %rbx +; SSSE3-NEXT: popq %r12 +; SSSE3-NEXT: popq %r13 +; SSSE3-NEXT: popq %r14 +; SSSE3-NEXT: popq %r15 +; SSSE3-NEXT: popq %rbp +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_sext_32i1_to_32i8: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: movswq (%rdi), %rax +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $62, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: movq %rax, %rdx +; SSE41-NEXT: shlq $63, %rdx +; SSE41-NEXT: sarq $63, %rdx +; SSE41-NEXT: movd %edx, %xmm0 +; SSE41-NEXT: pinsrb $1, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $61, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $2, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $60, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $3, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $59, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $4, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $58, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $5, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $57, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $6, %ecx, %xmm0 +; SSE41-NEXT: movsbq %al, %rcx +; SSE41-NEXT: shrq $7, %rcx +; SSE41-NEXT: pinsrb $7, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $55, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $8, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $54, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $9, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $53, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $10, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $52, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $11, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $51, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $12, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $50, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $13, %ecx, %xmm0 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $49, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $14, %ecx, %xmm0 +; SSE41-NEXT: shrq $15, %rax +; SSE41-NEXT: pinsrb $15, %eax, %xmm0 +; SSE41-NEXT: movswq 2(%rdi), %rax +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $62, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: movq %rax, %rdx +; SSE41-NEXT: shlq $63, %rdx +; SSE41-NEXT: sarq $63, %rdx +; SSE41-NEXT: movd %edx, %xmm1 +; SSE41-NEXT: pinsrb $1, %ecx, %xmm1 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $61, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $2, %ecx, %xmm1 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $60, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $3, %ecx, %xmm1 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $59, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $4, %ecx, %xmm1 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $58, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $5, %ecx, %xmm1 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $57, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $6, %ecx, %xmm1 +; SSE41-NEXT: movsbq %al, %rcx +; SSE41-NEXT: shrq $7, %rcx +; SSE41-NEXT: pinsrb $7, %ecx, %xmm1 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $55, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $8, %ecx, %xmm1 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $54, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $9, %ecx, %xmm1 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $53, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $10, %ecx, %xmm1 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $52, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $11, %ecx, %xmm1 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $51, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $12, %ecx, %xmm1 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $50, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $13, %ecx, %xmm1 +; SSE41-NEXT: movq %rax, %rcx +; SSE41-NEXT: shlq $49, %rcx +; SSE41-NEXT: sarq $63, %rcx +; SSE41-NEXT: pinsrb $14, %ecx, %xmm1 +; SSE41-NEXT: shrq $15, %rax +; SSE41-NEXT: pinsrb $15, %eax, %xmm1 +; SSE41-NEXT: retq +; +; AVX1-LABEL: load_sext_32i1_to_32i8: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: pushq %rbp +; AVX1-NEXT: pushq %r15 +; AVX1-NEXT: pushq %r14 +; AVX1-NEXT: pushq %r13 +; AVX1-NEXT: pushq %r12 +; AVX1-NEXT: pushq %rbx +; AVX1-NEXT: movslq (%rdi), %rax +; AVX1-NEXT: movq %rax, %rcx +; AVX1-NEXT: shlq $47, %rcx +; AVX1-NEXT: sarq $63, %rcx +; AVX1-NEXT: vmovd %ecx, %xmm0 +; AVX1-NEXT: movq %rax, %r8 +; AVX1-NEXT: movq %rax, %rdx +; AVX1-NEXT: movq %rax, %rcx +; AVX1-NEXT: movq %rax, %rdi +; AVX1-NEXT: movq %rax, %r13 +; AVX1-NEXT: movq %rax, %rsi +; AVX1-NEXT: movq %rax, %r10 +; AVX1-NEXT: movq %rax, %r11 +; AVX1-NEXT: movq %rax, %r9 +; AVX1-NEXT: movq %rax, %rbx +; AVX1-NEXT: movq %rax, %r14 +; AVX1-NEXT: movq %rax, %r15 +; AVX1-NEXT: movq %rax, %r12 +; AVX1-NEXT: movq %rax, %rbp +; AVX1-NEXT: shlq $46, %rbp +; AVX1-NEXT: sarq $63, %rbp +; AVX1-NEXT: vpinsrb $1, %ebp, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %rbp +; AVX1-NEXT: shlq $45, %r8 +; AVX1-NEXT: sarq $63, %r8 +; AVX1-NEXT: vpinsrb $2, %r8d, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %r8 +; AVX1-NEXT: shlq $44, %rdx +; AVX1-NEXT: sarq $63, %rdx +; AVX1-NEXT: vpinsrb $3, %edx, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %rdx +; AVX1-NEXT: shlq $43, %rcx +; AVX1-NEXT: sarq $63, %rcx +; AVX1-NEXT: vpinsrb $4, %ecx, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %rcx +; AVX1-NEXT: shlq $42, %rdi +; AVX1-NEXT: sarq $63, %rdi +; AVX1-NEXT: vpinsrb $5, %edi, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %rdi +; AVX1-NEXT: shlq $41, %r13 +; AVX1-NEXT: sarq $63, %r13 +; AVX1-NEXT: vpinsrb $6, %r13d, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %r13 +; AVX1-NEXT: shlq $40, %rsi +; AVX1-NEXT: sarq $63, %rsi +; AVX1-NEXT: vpinsrb $7, %esi, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %rsi +; AVX1-NEXT: shlq $39, %r10 +; AVX1-NEXT: sarq $63, %r10 +; AVX1-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %r10 +; AVX1-NEXT: shlq $38, %r11 +; AVX1-NEXT: sarq $63, %r11 +; AVX1-NEXT: vpinsrb $9, %r11d, %xmm0, %xmm0 +; AVX1-NEXT: movsbq %al, %r11 +; AVX1-NEXT: shlq $37, %r9 +; AVX1-NEXT: sarq $63, %r9 +; AVX1-NEXT: vpinsrb $10, %r9d, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %r9 +; AVX1-NEXT: shlq $36, %rbx +; AVX1-NEXT: sarq $63, %rbx +; AVX1-NEXT: vpinsrb $11, %ebx, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %rbx +; AVX1-NEXT: shlq $35, %r14 +; AVX1-NEXT: sarq $63, %r14 +; AVX1-NEXT: vpinsrb $12, %r14d, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %r14 +; AVX1-NEXT: shlq $34, %r15 +; AVX1-NEXT: sarq $63, %r15 +; AVX1-NEXT: vpinsrb $13, %r15d, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %r15 +; AVX1-NEXT: shlq $33, %r12 +; AVX1-NEXT: sarq $63, %r12 +; AVX1-NEXT: vpinsrb $14, %r12d, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %r12 +; AVX1-NEXT: shrq $31, %rbp +; AVX1-NEXT: vpinsrb $15, %ebp, %xmm0, %xmm0 +; AVX1-NEXT: movq %rax, %rbp +; AVX1-NEXT: shlq $63, %rdx +; AVX1-NEXT: sarq $63, %rdx +; AVX1-NEXT: vmovd %edx, %xmm1 +; AVX1-NEXT: movq %rax, %rdx +; AVX1-NEXT: movswq %ax, %rax +; AVX1-NEXT: shlq $62, %r8 +; AVX1-NEXT: sarq $63, %r8 +; AVX1-NEXT: vpinsrb $1, %r8d, %xmm1, %xmm1 +; AVX1-NEXT: shlq $61, %rcx +; AVX1-NEXT: sarq $63, %rcx +; AVX1-NEXT: vpinsrb $2, %ecx, %xmm1, %xmm1 +; AVX1-NEXT: shlq $60, %rdi +; AVX1-NEXT: sarq $63, %rdi +; AVX1-NEXT: vpinsrb $3, %edi, %xmm1, %xmm1 +; AVX1-NEXT: shlq $59, %r13 +; AVX1-NEXT: sarq $63, %r13 +; AVX1-NEXT: vpinsrb $4, %r13d, %xmm1, %xmm1 +; AVX1-NEXT: shlq $58, %rsi +; AVX1-NEXT: sarq $63, %rsi +; AVX1-NEXT: vpinsrb $5, %esi, %xmm1, %xmm1 +; AVX1-NEXT: shlq $57, %r10 +; AVX1-NEXT: sarq $63, %r10 +; AVX1-NEXT: vpinsrb $6, %r10d, %xmm1, %xmm1 +; AVX1-NEXT: shrq $7, %r11 +; AVX1-NEXT: vpinsrb $7, %r11d, %xmm1, %xmm1 +; AVX1-NEXT: shlq $55, %r9 +; AVX1-NEXT: sarq $63, %r9 +; AVX1-NEXT: vpinsrb $8, %r9d, %xmm1, %xmm1 +; AVX1-NEXT: shlq $54, %rbx +; AVX1-NEXT: sarq $63, %rbx +; AVX1-NEXT: vpinsrb $9, %ebx, %xmm1, %xmm1 +; AVX1-NEXT: shlq $53, %r14 +; AVX1-NEXT: sarq $63, %r14 +; AVX1-NEXT: vpinsrb $10, %r14d, %xmm1, %xmm1 +; AVX1-NEXT: shlq $52, %r15 +; AVX1-NEXT: sarq $63, %r15 +; AVX1-NEXT: vpinsrb $11, %r15d, %xmm1, %xmm1 +; AVX1-NEXT: shlq $51, %r12 +; AVX1-NEXT: sarq $63, %r12 +; AVX1-NEXT: vpinsrb $12, %r12d, %xmm1, %xmm1 +; AVX1-NEXT: shlq $50, %rbp +; AVX1-NEXT: sarq $63, %rbp +; AVX1-NEXT: vpinsrb $13, %ebp, %xmm1, %xmm1 +; AVX1-NEXT: shlq $49, %rdx +; AVX1-NEXT: sarq $63, %rdx +; AVX1-NEXT: vpinsrb $14, %edx, %xmm1, %xmm1 +; AVX1-NEXT: shrq $15, %rax +; AVX1-NEXT: vpinsrb $15, %eax, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: popq %rbx +; AVX1-NEXT: popq %r12 +; AVX1-NEXT: popq %r13 +; AVX1-NEXT: popq %r14 +; AVX1-NEXT: popq %r15 +; AVX1-NEXT: popq %rbp +; AVX1-NEXT: retq +; +; AVX2-LABEL: load_sext_32i1_to_32i8: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: pushq %rbp +; AVX2-NEXT: pushq %r15 +; AVX2-NEXT: pushq %r14 +; AVX2-NEXT: pushq %r13 +; AVX2-NEXT: pushq %r12 +; AVX2-NEXT: pushq %rbx +; AVX2-NEXT: movslq (%rdi), %rax +; AVX2-NEXT: movq %rax, %rcx +; AVX2-NEXT: shlq $47, %rcx +; AVX2-NEXT: sarq $63, %rcx +; AVX2-NEXT: vmovd %ecx, %xmm0 +; AVX2-NEXT: movq %rax, %r8 +; AVX2-NEXT: movq %rax, %rdx +; AVX2-NEXT: movq %rax, %rcx +; AVX2-NEXT: movq %rax, %rdi +; AVX2-NEXT: movq %rax, %r13 +; AVX2-NEXT: movq %rax, %rsi +; AVX2-NEXT: movq %rax, %r10 +; AVX2-NEXT: movq %rax, %r11 +; AVX2-NEXT: movq %rax, %r9 +; AVX2-NEXT: movq %rax, %rbx +; AVX2-NEXT: movq %rax, %r14 +; AVX2-NEXT: movq %rax, %r15 +; AVX2-NEXT: movq %rax, %r12 +; AVX2-NEXT: movq %rax, %rbp +; AVX2-NEXT: shlq $46, %rbp +; AVX2-NEXT: sarq $63, %rbp +; AVX2-NEXT: vpinsrb $1, %ebp, %xmm0, %xmm0 +; AVX2-NEXT: movq %rax, %rbp +; AVX2-NEXT: shlq $45, %r8 +; AVX2-NEXT: sarq $63, %r8 +; AVX2-NEXT: vpinsrb $2, %r8d, %xmm0, %xmm0 +; AVX2-NEXT: movq %rax, %r8 +; AVX2-NEXT: shlq $44, %rdx +; AVX2-NEXT: sarq $63, %rdx +; AVX2-NEXT: vpinsrb $3, %edx, %xmm0, %xmm0 +; AVX2-NEXT: movq %rax, %rdx +; AVX2-NEXT: shlq $43, %rcx +; AVX2-NEXT: sarq $63, %rcx +; AVX2-NEXT: vpinsrb $4, %ecx, %xmm0, %xmm0 +; AVX2-NEXT: movq %rax, %rcx +; AVX2-NEXT: shlq $42, %rdi +; AVX2-NEXT: sarq $63, %rdi +; AVX2-NEXT: vpinsrb $5, %edi, %xmm0, %xmm0 +; AVX2-NEXT: movq %rax, %rdi +; AVX2-NEXT: shlq $41, %r13 +; AVX2-NEXT: sarq $63, %r13 +; AVX2-NEXT: vpinsrb $6, %r13d, %xmm0, %xmm0 +; AVX2-NEXT: movq %rax, %r13 +; AVX2-NEXT: shlq $40, %rsi +; AVX2-NEXT: sarq $63, %rsi +; AVX2-NEXT: vpinsrb $7, %esi, %xmm0, %xmm0 +; AVX2-NEXT: movq %rax, %rsi +; AVX2-NEXT: shlq $39, %r10 +; AVX2-NEXT: sarq $63, %r10 +; AVX2-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0 +; AVX2-NEXT: movq %rax, %r10 +; AVX2-NEXT: shlq $38, %r11 +; AVX2-NEXT: sarq $63, %r11 +; AVX2-NEXT: vpinsrb $9, %r11d, %xmm0, %xmm0 +; AVX2-NEXT: movsbq %al, %r11 +; AVX2-NEXT: shlq $37, %r9 +; AVX2-NEXT: sarq $63, %r9 +; AVX2-NEXT: vpinsrb $10, %r9d, %xmm0, %xmm0 +; AVX2-NEXT: movq %rax, %r9 +; AVX2-NEXT: shlq $36, %rbx +; AVX2-NEXT: sarq $63, %rbx +; AVX2-NEXT: vpinsrb $11, %ebx, %xmm0, %xmm0 +; AVX2-NEXT: movq %rax, %rbx +; AVX2-NEXT: shlq $35, %r14 +; AVX2-NEXT: sarq $63, %r14 +; AVX2-NEXT: vpinsrb $12, %r14d, %xmm0, %xmm0 +; AVX2-NEXT: movq %rax, %r14 +; AVX2-NEXT: shlq $34, %r15 +; AVX2-NEXT: sarq $63, %r15 +; AVX2-NEXT: vpinsrb $13, %r15d, %xmm0, %xmm0 +; AVX2-NEXT: movq %rax, %r15 +; AVX2-NEXT: shlq $33, %r12 +; AVX2-NEXT: sarq $63, %r12 +; AVX2-NEXT: vpinsrb $14, %r12d, %xmm0, %xmm0 +; AVX2-NEXT: movq %rax, %r12 +; AVX2-NEXT: shrq $31, %rbp +; AVX2-NEXT: vpinsrb $15, %ebp, %xmm0, %xmm0 +; AVX2-NEXT: movq %rax, %rbp +; AVX2-NEXT: shlq $63, %rdx +; AVX2-NEXT: sarq $63, %rdx +; AVX2-NEXT: vmovd %edx, %xmm1 +; AVX2-NEXT: movq %rax, %rdx +; AVX2-NEXT: movswq %ax, %rax +; AVX2-NEXT: shlq $62, %r8 +; AVX2-NEXT: sarq $63, %r8 +; AVX2-NEXT: vpinsrb $1, %r8d, %xmm1, %xmm1 +; AVX2-NEXT: shlq $61, %rcx +; AVX2-NEXT: sarq $63, %rcx +; AVX2-NEXT: vpinsrb $2, %ecx, %xmm1, %xmm1 +; AVX2-NEXT: shlq $60, %rdi +; AVX2-NEXT: sarq $63, %rdi +; AVX2-NEXT: vpinsrb $3, %edi, %xmm1, %xmm1 +; AVX2-NEXT: shlq $59, %r13 +; AVX2-NEXT: sarq $63, %r13 +; AVX2-NEXT: vpinsrb $4, %r13d, %xmm1, %xmm1 +; AVX2-NEXT: shlq $58, %rsi +; AVX2-NEXT: sarq $63, %rsi +; AVX2-NEXT: vpinsrb $5, %esi, %xmm1, %xmm1 +; AVX2-NEXT: shlq $57, %r10 +; AVX2-NEXT: sarq $63, %r10 +; AVX2-NEXT: vpinsrb $6, %r10d, %xmm1, %xmm1 +; AVX2-NEXT: shrq $7, %r11 +; AVX2-NEXT: vpinsrb $7, %r11d, %xmm1, %xmm1 +; AVX2-NEXT: shlq $55, %r9 +; AVX2-NEXT: sarq $63, %r9 +; AVX2-NEXT: vpinsrb $8, %r9d, %xmm1, %xmm1 +; AVX2-NEXT: shlq $54, %rbx +; AVX2-NEXT: sarq $63, %rbx +; AVX2-NEXT: vpinsrb $9, %ebx, %xmm1, %xmm1 +; AVX2-NEXT: shlq $53, %r14 +; AVX2-NEXT: sarq $63, %r14 +; AVX2-NEXT: vpinsrb $10, %r14d, %xmm1, %xmm1 +; AVX2-NEXT: shlq $52, %r15 +; AVX2-NEXT: sarq $63, %r15 +; AVX2-NEXT: vpinsrb $11, %r15d, %xmm1, %xmm1 +; AVX2-NEXT: shlq $51, %r12 +; AVX2-NEXT: sarq $63, %r12 +; AVX2-NEXT: vpinsrb $12, %r12d, %xmm1, %xmm1 +; AVX2-NEXT: shlq $50, %rbp +; AVX2-NEXT: sarq $63, %rbp +; AVX2-NEXT: vpinsrb $13, %ebp, %xmm1, %xmm1 +; AVX2-NEXT: shlq $49, %rdx +; AVX2-NEXT: sarq $63, %rdx +; AVX2-NEXT: vpinsrb $14, %edx, %xmm1, %xmm1 +; AVX2-NEXT: shrq $15, %rax +; AVX2-NEXT: vpinsrb $15, %eax, %xmm1, %xmm1 +; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX2-NEXT: popq %rbx +; AVX2-NEXT: popq %r12 +; AVX2-NEXT: popq %r13 +; AVX2-NEXT: popq %r14 +; AVX2-NEXT: popq %r15 +; AVX2-NEXT: popq %rbp +; AVX2-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_32i1_to_32i8: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: pushl %esi +; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-SSE41-NEXT: movswl (%eax), %ecx +; X32-SSE41-NEXT: movl %ecx, %edx +; X32-SSE41-NEXT: shll $30, %edx +; X32-SSE41-NEXT: sarl $31, %edx +; X32-SSE41-NEXT: movl %ecx, %esi +; X32-SSE41-NEXT: shll $31, %esi +; X32-SSE41-NEXT: sarl $31, %esi +; X32-SSE41-NEXT: movd %esi, %xmm0 +; X32-SSE41-NEXT: pinsrb $1, %edx, %xmm0 +; X32-SSE41-NEXT: movl %ecx, %edx +; X32-SSE41-NEXT: shll $29, %edx +; X32-SSE41-NEXT: sarl $31, %edx +; X32-SSE41-NEXT: pinsrb $2, %edx, %xmm0 +; X32-SSE41-NEXT: movl %ecx, %edx +; X32-SSE41-NEXT: shll $28, %edx +; X32-SSE41-NEXT: sarl $31, %edx +; X32-SSE41-NEXT: pinsrb $3, %edx, %xmm0 +; X32-SSE41-NEXT: movl %ecx, %edx +; X32-SSE41-NEXT: shll $27, %edx +; X32-SSE41-NEXT: sarl $31, %edx +; X32-SSE41-NEXT: pinsrb $4, %edx, %xmm0 +; X32-SSE41-NEXT: movl %ecx, %edx +; X32-SSE41-NEXT: shll $26, %edx +; X32-SSE41-NEXT: sarl $31, %edx +; X32-SSE41-NEXT: pinsrb $5, %edx, %xmm0 +; X32-SSE41-NEXT: movl %ecx, %edx +; X32-SSE41-NEXT: shll $25, %edx +; X32-SSE41-NEXT: sarl $31, %edx +; X32-SSE41-NEXT: pinsrb $6, %edx, %xmm0 +; X32-SSE41-NEXT: movsbl %cl, %edx +; X32-SSE41-NEXT: shrl $7, %edx +; X32-SSE41-NEXT: pinsrb $7, %edx, %xmm0 +; X32-SSE41-NEXT: movl %ecx, %edx +; X32-SSE41-NEXT: shll $23, %edx +; X32-SSE41-NEXT: sarl $31, %edx +; X32-SSE41-NEXT: pinsrb $8, %edx, %xmm0 +; X32-SSE41-NEXT: movl %ecx, %edx +; X32-SSE41-NEXT: shll $22, %edx +; X32-SSE41-NEXT: sarl $31, %edx +; X32-SSE41-NEXT: pinsrb $9, %edx, %xmm0 +; X32-SSE41-NEXT: movl %ecx, %edx +; X32-SSE41-NEXT: shll $21, %edx +; X32-SSE41-NEXT: sarl $31, %edx +; X32-SSE41-NEXT: pinsrb $10, %edx, %xmm0 +; X32-SSE41-NEXT: movl %ecx, %edx +; X32-SSE41-NEXT: shll $20, %edx +; X32-SSE41-NEXT: sarl $31, %edx +; X32-SSE41-NEXT: pinsrb $11, %edx, %xmm0 +; X32-SSE41-NEXT: movl %ecx, %edx +; X32-SSE41-NEXT: shll $19, %edx +; X32-SSE41-NEXT: sarl $31, %edx +; X32-SSE41-NEXT: pinsrb $12, %edx, %xmm0 +; X32-SSE41-NEXT: movl %ecx, %edx +; X32-SSE41-NEXT: shll $18, %edx +; X32-SSE41-NEXT: sarl $31, %edx +; X32-SSE41-NEXT: pinsrb $13, %edx, %xmm0 +; X32-SSE41-NEXT: movl %ecx, %edx +; X32-SSE41-NEXT: shll $17, %edx +; X32-SSE41-NEXT: sarl $31, %edx +; X32-SSE41-NEXT: pinsrb $14, %edx, %xmm0 +; X32-SSE41-NEXT: shrl $15, %ecx +; X32-SSE41-NEXT: pinsrb $15, %ecx, %xmm0 +; X32-SSE41-NEXT: movswl 2(%eax), %eax +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $30, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: movl %eax, %edx +; X32-SSE41-NEXT: shll $31, %edx +; X32-SSE41-NEXT: sarl $31, %edx +; X32-SSE41-NEXT: movd %edx, %xmm1 +; X32-SSE41-NEXT: pinsrb $1, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $29, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $2, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $28, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $3, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $27, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $4, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $26, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $5, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $25, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $6, %ecx, %xmm1 +; X32-SSE41-NEXT: movsbl %al, %ecx +; X32-SSE41-NEXT: shrl $7, %ecx +; X32-SSE41-NEXT: pinsrb $7, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $23, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $8, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $22, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $9, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $21, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $10, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $20, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $11, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $19, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $12, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $18, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $13, %ecx, %xmm1 +; X32-SSE41-NEXT: movl %eax, %ecx +; X32-SSE41-NEXT: shll $17, %ecx +; X32-SSE41-NEXT: sarl $31, %ecx +; X32-SSE41-NEXT: pinsrb $14, %ecx, %xmm1 +; X32-SSE41-NEXT: shrl $15, %eax +; X32-SSE41-NEXT: pinsrb $15, %eax, %xmm1 +; X32-SSE41-NEXT: popl %esi +; X32-SSE41-NEXT: retl +entry: + %X = load <32 x i1>, <32 x i1>* %ptr + %Y = sext <32 x i1> %X to <32 x i8> + ret <32 x i8> %Y +} + +define <16 x i16> @load_sext_16i8_to_16i16(<16 x i8> *%ptr) { +; SSE2-LABEL: load_sext_16i8_to_16i16: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: psraw $8, %xmm0 +; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: psraw $8, %xmm1 +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_sext_16i8_to_16i16: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSSE3-NEXT: psraw $8, %xmm0 +; SSSE3-NEXT: movq {{.*#+}} xmm1 = mem[0],zero +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSSE3-NEXT: psraw $8, %xmm1 +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_sext_16i8_to_16i16: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovsxbw (%rdi), %xmm0 +; SSE41-NEXT: pmovsxbw 8(%rdi), %xmm1 +; SSE41-NEXT: retq +; +; AVX1-LABEL: load_sext_16i8_to_16i16: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpmovsxbw (%rdi), %xmm0 +; AVX1-NEXT: vpmovsxbw 8(%rdi), %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: load_sext_16i8_to_16i16: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovsxbw (%rdi), %ymm0 +; AVX2-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_16i8_to_16i16: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-SSE41-NEXT: pmovsxbw (%eax), %xmm0 +; X32-SSE41-NEXT: pmovsxbw 8(%eax), %xmm1 +; X32-SSE41-NEXT: retl +entry: + %X = load <16 x i8>, <16 x i8>* %ptr + %Y = sext <16 x i8> %X to <16 x i16> + ret <16 x i16> %Y +} + +define <2 x i64> @load_sext_2i16_to_2i64(<2 x i16> *%ptr) { +; SSE2-LABEL: load_sext_2i16_to_2i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: psrad $31, %xmm1 +; SSE2-NEXT: psrad $16, %xmm0 +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_sext_2i16_to_2i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: psrad $31, %xmm1 +; SSSE3-NEXT: psrad $16, %xmm0 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_sext_2i16_to_2i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovsxwq (%rdi), %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: load_sext_2i16_to_2i64: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovsxwq (%rdi), %xmm0 +; AVX-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_2i16_to_2i64: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-SSE41-NEXT: pmovsxwq (%eax), %xmm0 +; X32-SSE41-NEXT: retl +entry: + %X = load <2 x i16>, <2 x i16>* %ptr + %Y = sext <2 x i16> %X to <2 x i64> + ret <2 x i64> %Y +} + +define <4 x i32> @load_sext_4i16_to_4i32(<4 x i16> *%ptr) { +; SSE2-LABEL: load_sext_4i16_to_4i32: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE2-NEXT: psrad $16, %xmm0 +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_sext_4i16_to_4i32: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSSE3-NEXT: psrad $16, %xmm0 +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_sext_4i16_to_4i32: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovsxwd (%rdi), %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: load_sext_4i16_to_4i32: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovsxwd (%rdi), %xmm0 +; AVX-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_4i16_to_4i32: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-SSE41-NEXT: pmovsxwd (%eax), %xmm0 +; X32-SSE41-NEXT: retl +entry: + %X = load <4 x i16>, <4 x i16>* %ptr + %Y = sext <4 x i16> %X to <4 x i32> + ret <4 x i32> %Y +} + +define <4 x i64> @load_sext_4i16_to_4i64(<4 x i16> *%ptr) { +; SSE2-LABEL: load_sext_4i16_to_4i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movswq 2(%rdi), %rax +; SSE2-NEXT: movd %rax, %xmm1 +; SSE2-NEXT: movswq (%rdi), %rax +; SSE2-NEXT: movd %rax, %xmm0 +; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE2-NEXT: movswq 6(%rdi), %rax +; SSE2-NEXT: movd %rax, %xmm2 +; SSE2-NEXT: movswq 4(%rdi), %rax +; SSE2-NEXT: movd %rax, %xmm1 +; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_sext_4i16_to_4i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movswq 2(%rdi), %rax +; SSSE3-NEXT: movd %rax, %xmm1 +; SSSE3-NEXT: movswq (%rdi), %rax +; SSSE3-NEXT: movd %rax, %xmm0 +; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSSE3-NEXT: movswq 6(%rdi), %rax +; SSSE3-NEXT: movd %rax, %xmm2 +; SSSE3-NEXT: movswq 4(%rdi), %rax +; SSSE3-NEXT: movd %rax, %xmm1 +; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_sext_4i16_to_4i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovsxwq (%rdi), %xmm0 +; SSE41-NEXT: pmovsxwq 4(%rdi), %xmm1 +; SSE41-NEXT: retq +; +; AVX1-LABEL: load_sext_4i16_to_4i64: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpmovsxwd (%rdi), %xmm0 +; AVX1-NEXT: vpmovsxdq %xmm0, %xmm1 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; AVX1-NEXT: vpmovsxdq %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: load_sext_4i16_to_4i64: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovsxwq (%rdi), %ymm0 +; AVX2-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_4i16_to_4i64: +; X32-SSE41: # BB#0: # %entry +; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax +; X32-SSE41-NEXT: pmovsxwq (%eax), %xmm0 +; X32-SSE41-NEXT: pmovsxwq 4(%eax), %xmm1 +; X32-SSE41-NEXT: retl +entry: + %X = load <4 x i16>, <4 x i16>* %ptr + %Y = sext <4 x i16> %X to <4 x i64> + ret <4 x i64> %Y +} + +define <8 x i32> @load_sext_8i16_to_8i32(<8 x i16> *%ptr) { +; SSE2-LABEL: load_sext_8i16_to_8i32: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE2-NEXT: psrad $16, %xmm0 +; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] +; SSE2-NEXT: psrad $16, %xmm1 +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_sext_8i16_to_8i32: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSSE3-NEXT: psrad $16, %xmm0 +; SSSE3-NEXT: movq {{.*#+}} xmm1 = mem[0],zero +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] +; SSSE3-NEXT: psrad $16, %xmm1 +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_sext_8i16_to_8i32: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovsxwd (%rdi), %xmm0 +; SSE41-NEXT: pmovsxwd 8(%rdi), %xmm1 +; SSE41-NEXT: retq +; +; AVX1-LABEL: load_sext_8i16_to_8i32: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpmovsxwd (%rdi), %xmm0 +; AVX1-NEXT: vpmovsxwd 8(%rdi), %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: load_sext_8i16_to_8i32: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovsxwd (%rdi), %ymm0 +; AVX2-NEXT: retq ; -; X32-SSE41-LABEL: load_sext_test3: +; X32-SSE41-LABEL: load_sext_8i16_to_8i32: ; X32-SSE41: # BB#0: # %entry ; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-SSE41-NEXT: pmovsxbq (%eax), %xmm0 +; X32-SSE41-NEXT: pmovsxwd (%eax), %xmm0 +; X32-SSE41-NEXT: pmovsxwd 8(%eax), %xmm1 ; X32-SSE41-NEXT: retl entry: - %X = load <2 x i8>, <2 x i8>* %ptr - %Y = sext <2 x i8> %X to <2 x i64> - ret <2 x i64>%Y + %X = load <8 x i16>, <8 x i16>* %ptr + %Y = sext <8 x i16> %X to <8 x i32> + ret <8 x i32> %Y } -define <2 x i64> @load_sext_test4(<2 x i16> *%ptr) { -; SSE2-LABEL: load_sext_test4: +define <2 x i64> @load_sext_2i32_to_2i64(<2 x i32> *%ptr) { +; SSE2-LABEL: load_sext_2i32_to_2i64: ; SSE2: # BB#0: # %entry -; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero ; SSE2-NEXT: movdqa %xmm0, %xmm1 ; SSE2-NEXT: psrad $31, %xmm1 -; SSE2-NEXT: psrad $16, %xmm0 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; SSE2-NEXT: retq ; -; SSSE3-LABEL: load_sext_test4: +; SSSE3-LABEL: load_sext_2i32_to_2i64: ; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSSE3-NEXT: movq {{.*#+}} xmm0 = mem[0],zero ; SSSE3-NEXT: movdqa %xmm0, %xmm1 ; SSSE3-NEXT: psrad $31, %xmm1 -; SSSE3-NEXT: psrad $16, %xmm0 ; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; SSSE3-NEXT: retq ; -; SSE41-LABEL: load_sext_test4: +; SSE41-LABEL: load_sext_2i32_to_2i64: ; SSE41: # BB#0: # %entry -; SSE41-NEXT: pmovsxwq (%rdi), %xmm0 +; SSE41-NEXT: pmovsxdq (%rdi), %xmm0 ; SSE41-NEXT: retq ; -; AVX-LABEL: load_sext_test4: +; AVX-LABEL: load_sext_2i32_to_2i64: ; AVX: # BB#0: # %entry -; AVX-NEXT: vpmovsxwq (%rdi), %xmm0 +; AVX-NEXT: vpmovsxdq (%rdi), %xmm0 ; AVX-NEXT: retq ; -; X32-SSE41-LABEL: load_sext_test4: +; X32-SSE41-LABEL: load_sext_2i32_to_2i64: ; X32-SSE41: # BB#0: # %entry ; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-SSE41-NEXT: pmovsxwq (%eax), %xmm0 +; X32-SSE41-NEXT: pmovsxdq (%eax), %xmm0 ; X32-SSE41-NEXT: retl entry: - %X = load <2 x i16>, <2 x i16>* %ptr - %Y = sext <2 x i16> %X to <2 x i64> - ret <2 x i64>%Y + %X = load <2 x i32>, <2 x i32>* %ptr + %Y = sext <2 x i32> %X to <2 x i64> + ret <2 x i64> %Y } -define <2 x i64> @load_sext_test5(<2 x i32> *%ptr) { -; SSE2-LABEL: load_sext_test5: +define <4 x i64> @load_sext_4i32_to_4i64(<4 x i32> *%ptr) { +; SSE2-LABEL: load_sext_4i32_to_4i64: ; SSE2: # BB#0: # %entry -; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero -; SSE2-NEXT: movdqa %xmm0, %xmm1 -; SSE2-NEXT: psrad $31, %xmm1 -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE2-NEXT: movdqa (%rdi), %xmm0 +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: psrad $31, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: psrad $31, %xmm2 +; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] ; SSE2-NEXT: retq ; -; SSSE3-LABEL: load_sext_test5: +; SSSE3-LABEL: load_sext_4i32_to_4i64: ; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: movq {{.*#+}} xmm0 = mem[0],zero -; SSSE3-NEXT: movdqa %xmm0, %xmm1 -; SSSE3-NEXT: psrad $31, %xmm1 -; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSSE3-NEXT: movdqa (%rdi), %xmm0 +; SSSE3-NEXT: movdqa %xmm0, %xmm2 +; SSSE3-NEXT: psrad $31, %xmm2 +; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSSE3-NEXT: movdqa %xmm1, %xmm2 +; SSSE3-NEXT: psrad $31, %xmm2 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] ; SSSE3-NEXT: retq ; -; SSE41-LABEL: load_sext_test5: +; SSE41-LABEL: load_sext_4i32_to_4i64: ; SSE41: # BB#0: # %entry ; SSE41-NEXT: pmovsxdq (%rdi), %xmm0 +; SSE41-NEXT: pmovsxdq 8(%rdi), %xmm1 ; SSE41-NEXT: retq ; -; AVX-LABEL: load_sext_test5: -; AVX: # BB#0: # %entry -; AVX-NEXT: vpmovsxdq (%rdi), %xmm0 -; AVX-NEXT: retq +; AVX1-LABEL: load_sext_4i32_to_4i64: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpmovsxdq (%rdi), %xmm0 +; AVX1-NEXT: vpmovsxdq 8(%rdi), %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq ; -; X32-SSE41-LABEL: load_sext_test5: +; AVX2-LABEL: load_sext_4i32_to_4i64: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovsxdq (%rdi), %ymm0 +; AVX2-NEXT: retq +; +; X32-SSE41-LABEL: load_sext_4i32_to_4i64: ; X32-SSE41: # BB#0: # %entry ; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax ; X32-SSE41-NEXT: pmovsxdq (%eax), %xmm0 +; X32-SSE41-NEXT: pmovsxdq 8(%eax), %xmm1 ; X32-SSE41-NEXT: retl entry: - %X = load <2 x i32>, <2 x i32>* %ptr - %Y = sext <2 x i32> %X to <2 x i64> - ret <2 x i64>%Y + %X = load <4 x i32>, <4 x i32>* %ptr + %Y = sext <4 x i32> %X to <4 x i64> + ret <4 x i64> %Y } -define <8 x i16> @load_sext_test6(<8 x i8> *%ptr) { -; SSE2-LABEL: load_sext_test6: +define i32 @sext_2i8_to_i32(<16 x i8> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: sext_2i8_to_i32: ; SSE2: # BB#0: # %entry -; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero ; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] ; SSE2-NEXT: psraw $8, %xmm0 +; SSE2-NEXT: movd %xmm0, %eax ; SSE2-NEXT: retq ; -; SSSE3-LABEL: load_sext_test6: +; SSSE3-LABEL: sext_2i8_to_i32: ; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: movq {{.*#+}} xmm0 = mem[0],zero ; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] ; SSSE3-NEXT: psraw $8, %xmm0 +; SSSE3-NEXT: movd %xmm0, %eax ; SSSE3-NEXT: retq ; -; SSE41-LABEL: load_sext_test6: +; SSE41-LABEL: sext_2i8_to_i32: ; SSE41: # BB#0: # %entry -; SSE41-NEXT: pmovsxbw (%rdi), %xmm0 +; SSE41-NEXT: pmovsxbw %xmm0, %xmm0 +; SSE41-NEXT: movd %xmm0, %eax ; SSE41-NEXT: retq ; -; AVX-LABEL: load_sext_test6: +; AVX-LABEL: sext_2i8_to_i32: ; AVX: # BB#0: # %entry -; AVX-NEXT: vpmovsxbw (%rdi), %xmm0 +; AVX-NEXT: vpmovsxbw %xmm0, %xmm0 +; AVX-NEXT: vmovd %xmm0, %eax ; AVX-NEXT: retq ; -; X32-SSE41-LABEL: load_sext_test6: +; X32-SSE41-LABEL: sext_2i8_to_i32: ; X32-SSE41: # BB#0: # %entry -; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-SSE41-NEXT: pmovsxbw (%eax), %xmm0 +; X32-SSE41-NEXT: pushl %eax +; X32-SSE41-NEXT: .Ltmp0: +; X32-SSE41-NEXT: .cfi_def_cfa_offset 8 +; X32-SSE41-NEXT: pmovsxbw %xmm0, %xmm0 +; X32-SSE41-NEXT: movd %xmm0, %eax +; X32-SSE41-NEXT: popl %ecx ; X32-SSE41-NEXT: retl entry: - %X = load <8 x i8>, <8 x i8>* %ptr - %Y = sext <8 x i8> %X to <8 x i16> - ret <8 x i16>%Y + %Shuf = shufflevector <16 x i8> %A, <16 x i8> undef, <2 x i32> + %Ex = sext <2 x i8> %Shuf to <2 x i16> + %Bc = bitcast <2 x i16> %Ex to i32 + ret i32 %Bc } define <4 x i64> @sext_4i1_to_4i64(<4 x i1> %mask) { @@ -460,57 +3877,6 @@ define <4 x i64> @sext_4i1_to_4i64(<4 x i1> %mask) { ret <4 x i64> %extmask } -define <16 x i16> @sext_16i8_to_16i16(<16 x i8> *%ptr) { -; SSE2-LABEL: sext_16i8_to_16i16: -; SSE2: # BB#0: # %entry -; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE2-NEXT: psraw $8, %xmm0 -; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero -; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE2-NEXT: psraw $8, %xmm1 -; SSE2-NEXT: retq -; -; SSSE3-LABEL: sext_16i8_to_16i16: -; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: movq {{.*#+}} xmm0 = mem[0],zero -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSSE3-NEXT: psraw $8, %xmm0 -; SSSE3-NEXT: movq {{.*#+}} xmm1 = mem[0],zero -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSSE3-NEXT: psraw $8, %xmm1 -; SSSE3-NEXT: retq -; -; SSE41-LABEL: sext_16i8_to_16i16: -; SSE41: # BB#0: # %entry -; SSE41-NEXT: pmovsxbw (%rdi), %xmm0 -; SSE41-NEXT: pmovsxbw 8(%rdi), %xmm1 -; SSE41-NEXT: retq -; -; AVX1-LABEL: sext_16i8_to_16i16: -; AVX1: # BB#0: # %entry -; AVX1-NEXT: vpmovsxbw (%rdi), %xmm0 -; AVX1-NEXT: vpmovsxbw 8(%rdi), %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; AVX1-NEXT: retq -; -; AVX2-LABEL: sext_16i8_to_16i16: -; AVX2: # BB#0: # %entry -; AVX2-NEXT: vpmovsxbw (%rdi), %ymm0 -; AVX2-NEXT: retq -; -; X32-SSE41-LABEL: sext_16i8_to_16i16: -; X32-SSE41: # BB#0: # %entry -; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-SSE41-NEXT: pmovsxbw (%eax), %xmm0 -; X32-SSE41-NEXT: pmovsxbw 8(%eax), %xmm1 -; X32-SSE41-NEXT: retl -entry: - %X = load <16 x i8>, <16 x i8>* %ptr - %Y = sext <16 x i8> %X to <16 x i16> - ret <16 x i16> %Y -} - define <4 x i64> @sext_4i8_to_4i64(<4 x i8> %mask) { ; SSE2-LABEL: sext_4i8_to_4i64: ; SSE2: # BB#0: @@ -577,125 +3943,3 @@ define <4 x i64> @sext_4i8_to_4i64(<4 x i8> %mask) { %extmask = sext <4 x i8> %mask to <4 x i64> ret <4 x i64> %extmask } - -define <4 x i64> @load_sext_4i8_to_4i64(<4 x i8> *%ptr) { -; SSE2-LABEL: load_sext_4i8_to_4i64: -; SSE2: # BB#0: # %entry -; SSE2-NEXT: movsbq 1(%rdi), %rax -; SSE2-NEXT: movd %rax, %xmm1 -; SSE2-NEXT: movsbq (%rdi), %rax -; SSE2-NEXT: movd %rax, %xmm0 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] -; SSE2-NEXT: movsbq 3(%rdi), %rax -; SSE2-NEXT: movd %rax, %xmm2 -; SSE2-NEXT: movsbq 2(%rdi), %rax -; SSE2-NEXT: movd %rax, %xmm1 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] -; SSE2-NEXT: retq -; -; SSSE3-LABEL: load_sext_4i8_to_4i64: -; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: movsbq 1(%rdi), %rax -; SSSE3-NEXT: movd %rax, %xmm1 -; SSSE3-NEXT: movsbq (%rdi), %rax -; SSSE3-NEXT: movd %rax, %xmm0 -; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] -; SSSE3-NEXT: movsbq 3(%rdi), %rax -; SSSE3-NEXT: movd %rax, %xmm2 -; SSSE3-NEXT: movsbq 2(%rdi), %rax -; SSSE3-NEXT: movd %rax, %xmm1 -; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] -; SSSE3-NEXT: retq -; -; SSE41-LABEL: load_sext_4i8_to_4i64: -; SSE41: # BB#0: # %entry -; SSE41-NEXT: pmovsxbq (%rdi), %xmm0 -; SSE41-NEXT: pmovsxbq 2(%rdi), %xmm1 -; SSE41-NEXT: retq -; -; AVX1-LABEL: load_sext_4i8_to_4i64: -; AVX1: # BB#0: # %entry -; AVX1-NEXT: vpmovsxbd (%rdi), %xmm0 -; AVX1-NEXT: vpmovsxdq %xmm0, %xmm1 -; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; AVX1-NEXT: vpmovsxdq %xmm0, %xmm0 -; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 -; AVX1-NEXT: retq -; -; AVX2-LABEL: load_sext_4i8_to_4i64: -; AVX2: # BB#0: # %entry -; AVX2-NEXT: vpmovsxbq (%rdi), %ymm0 -; AVX2-NEXT: retq -; -; X32-SSE41-LABEL: load_sext_4i8_to_4i64: -; X32-SSE41: # BB#0: # %entry -; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-SSE41-NEXT: pmovsxbq (%eax), %xmm0 -; X32-SSE41-NEXT: pmovsxbq 2(%eax), %xmm1 -; X32-SSE41-NEXT: retl -entry: - %X = load <4 x i8>, <4 x i8>* %ptr - %Y = sext <4 x i8> %X to <4 x i64> - ret <4 x i64>%Y -} - -define <4 x i64> @load_sext_4i16_to_4i64(<4 x i16> *%ptr) { -; SSE2-LABEL: load_sext_4i16_to_4i64: -; SSE2: # BB#0: # %entry -; SSE2-NEXT: movswq 2(%rdi), %rax -; SSE2-NEXT: movd %rax, %xmm1 -; SSE2-NEXT: movswq (%rdi), %rax -; SSE2-NEXT: movd %rax, %xmm0 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] -; SSE2-NEXT: movswq 6(%rdi), %rax -; SSE2-NEXT: movd %rax, %xmm2 -; SSE2-NEXT: movswq 4(%rdi), %rax -; SSE2-NEXT: movd %rax, %xmm1 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] -; SSE2-NEXT: retq -; -; SSSE3-LABEL: load_sext_4i16_to_4i64: -; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: movswq 2(%rdi), %rax -; SSSE3-NEXT: movd %rax, %xmm1 -; SSSE3-NEXT: movswq (%rdi), %rax -; SSSE3-NEXT: movd %rax, %xmm0 -; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] -; SSSE3-NEXT: movswq 6(%rdi), %rax -; SSSE3-NEXT: movd %rax, %xmm2 -; SSSE3-NEXT: movswq 4(%rdi), %rax -; SSSE3-NEXT: movd %rax, %xmm1 -; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] -; SSSE3-NEXT: retq -; -; SSE41-LABEL: load_sext_4i16_to_4i64: -; SSE41: # BB#0: # %entry -; SSE41-NEXT: pmovsxwq (%rdi), %xmm0 -; SSE41-NEXT: pmovsxwq 4(%rdi), %xmm1 -; SSE41-NEXT: retq -; -; AVX1-LABEL: load_sext_4i16_to_4i64: -; AVX1: # BB#0: # %entry -; AVX1-NEXT: vpmovsxwd (%rdi), %xmm0 -; AVX1-NEXT: vpmovsxdq %xmm0, %xmm1 -; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; AVX1-NEXT: vpmovsxdq %xmm0, %xmm0 -; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 -; AVX1-NEXT: retq -; -; AVX2-LABEL: load_sext_4i16_to_4i64: -; AVX2: # BB#0: # %entry -; AVX2-NEXT: vpmovsxwq (%rdi), %ymm0 -; AVX2-NEXT: retq -; -; X32-SSE41-LABEL: load_sext_4i16_to_4i64: -; X32-SSE41: # BB#0: # %entry -; X32-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-SSE41-NEXT: pmovsxwq (%eax), %xmm0 -; X32-SSE41-NEXT: pmovsxwq 4(%eax), %xmm1 -; X32-SSE41-NEXT: retl -entry: - %X = load <4 x i16>, <4 x i16>* %ptr - %Y = sext <4 x i16> %X to <4 x i64> - ret <4 x i64>%Y -} diff --git a/test/CodeGen/X86/vector-shift-ashr-128.ll b/test/CodeGen/X86/vector-shift-ashr-128.ll index 61b30154950d..771445df85e0 100644 --- a/test/CodeGen/X86/vector-shift-ashr-128.ll +++ b/test/CodeGen/X86/vector-shift-ashr-128.ll @@ -1,59 +1,115 @@ -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX2 +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW + +; +; Just one 32-bit run to make sure we do reasonable things for i64 shifts. +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=X32-SSE --check-prefix=X32-SSE2 ; ; Variable Shifts ; -define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) { +define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { ; SSE2-LABEL: var_shift_v2i64: ; SSE2: # BB#0: -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: movd %xmm1, %rcx -; SSE2-NEXT: sarq %cl, %rax -; SSE2-NEXT: movd %rax, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] -; SSE2-NEXT: movd %xmm0, %rcx -; SSE2-NEXT: sarq %cl, %rax -; SSE2-NEXT: movd %rax, %xmm0 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0] +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1] +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: psrlq %xmm3, %xmm4 +; SSE2-NEXT: psrlq %xmm1, %xmm2 +; SSE2-NEXT: movsd {{.*#+}} xmm4 = xmm2[0],xmm4[1] +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: psrlq %xmm3, %xmm2 +; SSE2-NEXT: psrlq %xmm1, %xmm0 +; SSE2-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1] +; SSE2-NEXT: xorpd %xmm4, %xmm2 +; SSE2-NEXT: psubq %xmm4, %xmm2 ; SSE2-NEXT: movdqa %xmm2, %xmm0 ; SSE2-NEXT: retq ; ; SSE41-LABEL: var_shift_v2i64: ; SSE41: # BB#0: -; SSE41-NEXT: pextrq $1, %xmm0, %rax -; SSE41-NEXT: pextrq $1, %xmm1, %rcx -; SSE41-NEXT: sarq %cl, %rax -; SSE41-NEXT: movd %rax, %xmm2 -; SSE41-NEXT: movd %xmm0, %rax -; SSE41-NEXT: movd %xmm1, %rcx -; SSE41-NEXT: sarq %cl, %rax -; SSE41-NEXT: movd %rax, %xmm0 -; SSE41-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; SSE41-NEXT: movdqa %xmm2, %xmm3 +; SSE41-NEXT: psrlq %xmm1, %xmm3 +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1] +; SSE41-NEXT: psrlq %xmm4, %xmm2 +; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7] +; SSE41-NEXT: movdqa %xmm0, %xmm3 +; SSE41-NEXT: psrlq %xmm1, %xmm3 +; SSE41-NEXT: psrlq %xmm4, %xmm0 +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7] +; SSE41-NEXT: pxor %xmm2, %xmm0 +; SSE41-NEXT: psubq %xmm2, %xmm0 ; SSE41-NEXT: retq ; -; AVX-LABEL: var_shift_v2i64: -; AVX: # BB#0: -; AVX-NEXT: vpextrq $1, %xmm0, %rax -; AVX-NEXT: vpextrq $1, %xmm1, %rcx -; AVX-NEXT: sarq %cl, %rax -; AVX-NEXT: vmovq %rax, %xmm2 -; AVX-NEXT: vmovq %xmm0, %rax -; AVX-NEXT: vmovq %xmm1, %rcx -; AVX-NEXT: sarq %cl, %rax -; AVX-NEXT: vmovq %rax, %xmm0 -; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] -; AVX-NEXT: retq +; AVX1-LABEL: var_shift_v2i64: +; AVX1: # BB#0: +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vpsrlq %xmm1, %xmm2, %xmm3 +; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,3,0,1] +; AVX1-NEXT: vpsrlq %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vpsrlq %xmm1, %xmm0, %xmm1 +; AVX1-NEXT: vpsrlq %xmm4, %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7] +; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpsubq %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: var_shift_v2i64: +; AVX2: # BB#0: +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX2-NEXT: vpsrlvq %xmm1, %xmm2, %xmm3 +; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpsrlvq %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpsubq %xmm3, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; XOP-LABEL: var_shift_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOP-NEXT: vpsubq %xmm1, %xmm2, %xmm1 +; XOP-NEXT: vpshaq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: var_shift_v2i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX512-NEXT: vpsrlvq %xmm1, %xmm2, %xmm3 +; AVX512-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vpsrlvq %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpsubq %xmm3, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: var_shift_v2i64: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1] +; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [0,2147483648,0,2147483648] +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: psrlq %xmm2, %xmm4 +; X32-SSE-NEXT: movq {{.*#+}} xmm5 = xmm1[0],zero +; X32-SSE-NEXT: psrlq %xmm5, %xmm3 +; X32-SSE-NEXT: movsd {{.*#+}} xmm4 = xmm3[0],xmm4[1] +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psrlq %xmm2, %xmm1 +; X32-SSE-NEXT: psrlq %xmm5, %xmm0 +; X32-SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1] +; X32-SSE-NEXT: xorpd %xmm4, %xmm1 +; X32-SSE-NEXT: psubq %xmm4, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm0 +; X32-SSE-NEXT: retl %shift = ashr <2 x i64> %a, %b ret <2 x i64> %shift } -define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) { +define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { ; SSE2-LABEL: var_shift_v4i32: ; SSE2: # BB#0: ; SSE2-NEXT: movdqa %xmm1, %xmm2 @@ -119,11 +175,52 @@ define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsravd %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_shift_v4i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpsubd %xmm1, %xmm2, %xmm1 +; XOPAVX1-NEXT: vpshad %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_shift_v4i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsravd %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: var_shift_v4i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsravd %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: var_shift_v4i32: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: psrldq {{.*#+}} xmm2 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; X32-SSE-NEXT: movdqa %xmm0, %xmm3 +; X32-SSE-NEXT: psrad %xmm2, %xmm3 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: psrlq $32, %xmm2 +; X32-SSE-NEXT: movdqa %xmm0, %xmm4 +; X32-SSE-NEXT: psrad %xmm2, %xmm4 +; X32-SSE-NEXT: movsd {{.*#+}} xmm3 = xmm4[0],xmm3[1] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,3,2,3] +; X32-SSE-NEXT: pxor %xmm3, %xmm3 +; X32-SSE-NEXT: movdqa %xmm1, %xmm4 +; X32-SSE-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3] +; X32-SSE-NEXT: movdqa %xmm0, %xmm5 +; X32-SSE-NEXT: psrad %xmm4, %xmm5 +; X32-SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] +; X32-SSE-NEXT: psrad %xmm1, %xmm0 +; X32-SSE-NEXT: movsd {{.*#+}} xmm5 = xmm0[0],xmm5[1] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3] +; X32-SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; X32-SSE-NEXT: retl %shift = ashr <4 x i32> %a, %b ret <4 x i32> %shift } -define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { +define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { ; SSE2-LABEL: var_shift_v8i16: ; SSE2: # BB#0: ; SSE2-NEXT: psllw $12, %xmm1 @@ -216,11 +313,58 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq +; +; XOP-LABEL: var_shift_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOP-NEXT: vpsubw %xmm1, %xmm2, %xmm1 +; XOP-NEXT: vpshaw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: var_shift_v8i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsravw %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: var_shift_v8i16: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: psllw $12, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: psraw $15, %xmm2 +; X32-SSE-NEXT: movdqa %xmm2, %xmm3 +; X32-SSE-NEXT: pandn %xmm0, %xmm3 +; X32-SSE-NEXT: psraw $8, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: por %xmm3, %xmm0 +; X32-SSE-NEXT: paddw %xmm1, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: psraw $15, %xmm2 +; X32-SSE-NEXT: movdqa %xmm2, %xmm3 +; X32-SSE-NEXT: pandn %xmm0, %xmm3 +; X32-SSE-NEXT: psraw $4, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: por %xmm3, %xmm0 +; X32-SSE-NEXT: paddw %xmm1, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: psraw $15, %xmm2 +; X32-SSE-NEXT: movdqa %xmm2, %xmm3 +; X32-SSE-NEXT: pandn %xmm0, %xmm3 +; X32-SSE-NEXT: psraw $2, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: por %xmm3, %xmm0 +; X32-SSE-NEXT: paddw %xmm1, %xmm1 +; X32-SSE-NEXT: psraw $15, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: pandn %xmm0, %xmm2 +; X32-SSE-NEXT: psraw $1, %xmm0 +; X32-SSE-NEXT: pand %xmm1, %xmm0 +; X32-SSE-NEXT: por %xmm2, %xmm0 +; X32-SSE-NEXT: retl %shift = ashr <8 x i16> %a, %b ret <8 x i16> %shift } -define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { +define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { ; SSE2-LABEL: var_shift_v16i8: ; SSE2: # BB#0: ; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] @@ -342,6 +486,99 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX-NEXT: vpsrlw $8, %xmm0, %xmm0 ; AVX-NEXT: vpackuswb %xmm2, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: var_shift_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOP-NEXT: vpsubb %xmm1, %xmm2, %xmm1 +; XOP-NEXT: vpshab %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: var_shift_v16i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllw $5, %xmm1, %xmm1 +; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] +; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] +; AVX512-NEXT: vpsraw $4, %xmm3, %xmm4 +; AVX512-NEXT: vpblendvb %xmm2, %xmm4, %xmm3, %xmm3 +; AVX512-NEXT: vpsraw $2, %xmm3, %xmm4 +; AVX512-NEXT: vpaddw %xmm2, %xmm2, %xmm2 +; AVX512-NEXT: vpblendvb %xmm2, %xmm4, %xmm3, %xmm3 +; AVX512-NEXT: vpsraw $1, %xmm3, %xmm4 +; AVX512-NEXT: vpaddw %xmm2, %xmm2, %xmm2 +; AVX512-NEXT: vpblendvb %xmm2, %xmm4, %xmm3, %xmm2 +; AVX512-NEXT: vpsrlw $8, %xmm2, %xmm2 +; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; AVX512-NEXT: vpsraw $4, %xmm0, %xmm3 +; AVX512-NEXT: vpblendvb %xmm1, %xmm3, %xmm0, %xmm0 +; AVX512-NEXT: vpsraw $2, %xmm0, %xmm3 +; AVX512-NEXT: vpaddw %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm3, %xmm0, %xmm0 +; AVX512-NEXT: vpsraw $1, %xmm0, %xmm3 +; AVX512-NEXT: vpaddw %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm3, %xmm0, %xmm0 +; AVX512-NEXT: vpsrlw $8, %xmm0, %xmm0 +; AVX512-NEXT: vpackuswb %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: var_shift_v16i8: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] +; X32-SSE-NEXT: psllw $5, %xmm1 +; X32-SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15] +; X32-SSE-NEXT: pxor %xmm3, %xmm3 +; X32-SSE-NEXT: pxor %xmm5, %xmm5 +; X32-SSE-NEXT: pcmpgtw %xmm4, %xmm5 +; X32-SSE-NEXT: movdqa %xmm5, %xmm6 +; X32-SSE-NEXT: pandn %xmm2, %xmm6 +; X32-SSE-NEXT: psraw $4, %xmm2 +; X32-SSE-NEXT: pand %xmm5, %xmm2 +; X32-SSE-NEXT: por %xmm6, %xmm2 +; X32-SSE-NEXT: paddw %xmm4, %xmm4 +; X32-SSE-NEXT: pxor %xmm5, %xmm5 +; X32-SSE-NEXT: pcmpgtw %xmm4, %xmm5 +; X32-SSE-NEXT: movdqa %xmm5, %xmm6 +; X32-SSE-NEXT: pandn %xmm2, %xmm6 +; X32-SSE-NEXT: psraw $2, %xmm2 +; X32-SSE-NEXT: pand %xmm5, %xmm2 +; X32-SSE-NEXT: por %xmm6, %xmm2 +; X32-SSE-NEXT: paddw %xmm4, %xmm4 +; X32-SSE-NEXT: pxor %xmm5, %xmm5 +; X32-SSE-NEXT: pcmpgtw %xmm4, %xmm5 +; X32-SSE-NEXT: movdqa %xmm5, %xmm4 +; X32-SSE-NEXT: pandn %xmm2, %xmm4 +; X32-SSE-NEXT: psraw $1, %xmm2 +; X32-SSE-NEXT: pand %xmm5, %xmm2 +; X32-SSE-NEXT: por %xmm4, %xmm2 +; X32-SSE-NEXT: psrlw $8, %xmm2 +; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; X32-SSE-NEXT: pxor %xmm4, %xmm4 +; X32-SSE-NEXT: pcmpgtw %xmm1, %xmm4 +; X32-SSE-NEXT: movdqa %xmm4, %xmm5 +; X32-SSE-NEXT: pandn %xmm0, %xmm5 +; X32-SSE-NEXT: psraw $4, %xmm0 +; X32-SSE-NEXT: pand %xmm4, %xmm0 +; X32-SSE-NEXT: por %xmm5, %xmm0 +; X32-SSE-NEXT: paddw %xmm1, %xmm1 +; X32-SSE-NEXT: pxor %xmm4, %xmm4 +; X32-SSE-NEXT: pcmpgtw %xmm1, %xmm4 +; X32-SSE-NEXT: movdqa %xmm4, %xmm5 +; X32-SSE-NEXT: pandn %xmm0, %xmm5 +; X32-SSE-NEXT: psraw $2, %xmm0 +; X32-SSE-NEXT: pand %xmm4, %xmm0 +; X32-SSE-NEXT: por %xmm5, %xmm0 +; X32-SSE-NEXT: paddw %xmm1, %xmm1 +; X32-SSE-NEXT: pcmpgtw %xmm1, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm1 +; X32-SSE-NEXT: pandn %xmm0, %xmm1 +; X32-SSE-NEXT: psraw $1, %xmm0 +; X32-SSE-NEXT: pand %xmm3, %xmm0 +; X32-SSE-NEXT: por %xmm1, %xmm0 +; X32-SSE-NEXT: psrlw $8, %xmm0 +; X32-SSE-NEXT: packuswb %xmm2, %xmm0 +; X32-SSE-NEXT: retl %shift = ashr <16 x i8> %a, %b ret <16 x i8> %shift } @@ -350,71 +587,65 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; Uniform Variable Shifts ; -define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) { -; SSE2-LABEL: splatvar_shift_v2i64: -; SSE2: # BB#0: -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,1,0,1] -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: movd %xmm2, %rcx -; SSE2-NEXT: sarq %cl, %rax -; SSE2-NEXT: movd %rax, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1] -; SSE2-NEXT: movd %xmm0, %rcx -; SSE2-NEXT: sarq %cl, %rax -; SSE2-NEXT: movd %rax, %xmm0 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] -; SSE2-NEXT: movdqa %xmm1, %xmm0 -; SSE2-NEXT: retq +define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { +; SSE-LABEL: splatvar_shift_v2i64: +; SSE: # BB#0: +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; SSE-NEXT: psrlq %xmm1, %xmm2 +; SSE-NEXT: psrlq %xmm1, %xmm0 +; SSE-NEXT: pxor %xmm2, %xmm0 +; SSE-NEXT: psubq %xmm2, %xmm0 +; SSE-NEXT: retq ; -; SSE41-LABEL: splatvar_shift_v2i64: -; SSE41: # BB#0: -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1] -; SSE41-NEXT: pextrq $1, %xmm0, %rax -; SSE41-NEXT: pextrq $1, %xmm1, %rcx -; SSE41-NEXT: sarq %cl, %rax -; SSE41-NEXT: movd %rax, %xmm2 -; SSE41-NEXT: movd %xmm0, %rax -; SSE41-NEXT: movd %xmm1, %rcx -; SSE41-NEXT: sarq %cl, %rax -; SSE41-NEXT: movd %rax, %xmm0 -; SSE41-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] -; SSE41-NEXT: retq +; AVX-LABEL: splatvar_shift_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX-NEXT: vpsrlq %xmm1, %xmm2, %xmm2 +; AVX-NEXT: vpsrlq %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpsubq %xmm2, %xmm0, %xmm0 +; AVX-NEXT: retq ; -; AVX1-LABEL: splatvar_shift_v2i64: -; AVX1: # BB#0: -; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1] -; AVX1-NEXT: vpextrq $1, %xmm0, %rax -; AVX1-NEXT: vpextrq $1, %xmm1, %rcx -; AVX1-NEXT: sarq %cl, %rax -; AVX1-NEXT: vmovq %rax, %xmm2 -; AVX1-NEXT: vmovq %xmm0, %rax -; AVX1-NEXT: vmovq %xmm1, %rcx -; AVX1-NEXT: sarq %cl, %rax -; AVX1-NEXT: vmovq %rax, %xmm0 -; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] -; AVX1-NEXT: retq +; XOPAVX1-LABEL: splatvar_shift_v2i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1] +; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpsubq %xmm1, %xmm2, %xmm1 +; XOPAVX1-NEXT: vpshaq %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: retq ; -; AVX2-LABEL: splatvar_shift_v2i64: -; AVX2: # BB#0: -; AVX2-NEXT: vpbroadcastq %xmm1, %xmm1 -; AVX2-NEXT: vpextrq $1, %xmm0, %rax -; AVX2-NEXT: vpextrq $1, %xmm1, %rcx -; AVX2-NEXT: sarq %cl, %rax -; AVX2-NEXT: vmovq %rax, %xmm2 -; AVX2-NEXT: vmovq %xmm0, %rax -; AVX2-NEXT: vmovq %xmm1, %rcx -; AVX2-NEXT: sarq %cl, %rax -; AVX2-NEXT: vmovq %rax, %xmm0 -; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] -; AVX2-NEXT: retq +; XOPAVX2-LABEL: splatvar_shift_v2i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpbroadcastq %xmm1, %xmm1 +; XOPAVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX2-NEXT: vpsubq %xmm1, %xmm2, %xmm1 +; XOPAVX2-NEXT: vpshaq %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v2i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX512-NEXT: vpsrlq %xmm1, %xmm2, %xmm2 +; AVX512-NEXT: vpsrlq %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vpsubq %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatvar_shift_v2i64: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movq {{.*#+}} xmm1 = xmm1[0],zero +; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [0,2147483648,0,2147483648] +; X32-SSE-NEXT: psrlq %xmm1, %xmm2 +; X32-SSE-NEXT: psrlq %xmm1, %xmm0 +; X32-SSE-NEXT: pxor %xmm2, %xmm0 +; X32-SSE-NEXT: psubq %xmm2, %xmm0 +; X32-SSE-NEXT: retl %splat = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer %shift = ashr <2 x i64> %a, %splat ret <2 x i64> %shift } -define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) { +define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { ; SSE2-LABEL: splatvar_shift_v4i32: ; SSE2: # BB#0: ; SSE2-NEXT: xorps %xmm2, %xmm2 @@ -435,12 +666,33 @@ define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) { ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7] ; AVX-NEXT: vpsrad %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatvar_shift_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7] +; XOP-NEXT: vpsrad %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v4i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vxorps %xmm2, %xmm2, %xmm2 +; AVX512-NEXT: vmovss %xmm1, %xmm2, %xmm1 +; AVX512-NEXT: vpsrad %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatvar_shift_v4i32: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: xorps %xmm2, %xmm2 +; X32-SSE-NEXT: movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3] +; X32-SSE-NEXT: psrad %xmm2, %xmm0 +; X32-SSE-NEXT: retl %splat = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer %shift = ashr <4 x i32> %a, %splat ret <4 x i32> %shift } -define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { +define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { ; SSE2-LABEL: splatvar_shift_v8i16: ; SSE2: # BB#0: ; SSE2-NEXT: movd %xmm1, %eax @@ -462,12 +714,34 @@ define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7] ; AVX-NEXT: vpsraw %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatvar_shift_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7] +; XOP-NEXT: vpsraw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v8i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7] +; AVX512-NEXT: vpsraw %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatvar_shift_v8i16: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movd %xmm1, %eax +; X32-SSE-NEXT: movzwl %ax, %eax +; X32-SSE-NEXT: movd %eax, %xmm1 +; X32-SSE-NEXT: psraw %xmm1, %xmm0 +; X32-SSE-NEXT: retl %splat = shufflevector <8 x i16> %b, <8 x i16> undef, <8 x i32> zeroinitializer %shift = ashr <8 x i16> %a, %splat ret <8 x i16> %shift } -define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { +define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { ; SSE2-LABEL: splatvar_shift_v16i8: ; SSE2: # BB#0: ; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] @@ -626,6 +900,113 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0 ; AVX2-NEXT: vpackuswb %xmm2, %xmm0, %xmm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatvar_shift_v16i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpsubb %xmm1, %xmm2, %xmm1 +; XOPAVX1-NEXT: vpshab %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatvar_shift_v16i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpbroadcastb %xmm1, %xmm1 +; XOPAVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX2-NEXT: vpsubb %xmm1, %xmm2, %xmm1 +; XOPAVX2-NEXT: vpshab %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v16i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpbroadcastb %xmm1, %xmm1 +; AVX512-NEXT: vpsllw $5, %xmm1, %xmm1 +; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] +; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] +; AVX512-NEXT: vpsraw $4, %xmm3, %xmm4 +; AVX512-NEXT: vpblendvb %xmm2, %xmm4, %xmm3, %xmm3 +; AVX512-NEXT: vpsraw $2, %xmm3, %xmm4 +; AVX512-NEXT: vpaddw %xmm2, %xmm2, %xmm2 +; AVX512-NEXT: vpblendvb %xmm2, %xmm4, %xmm3, %xmm3 +; AVX512-NEXT: vpsraw $1, %xmm3, %xmm4 +; AVX512-NEXT: vpaddw %xmm2, %xmm2, %xmm2 +; AVX512-NEXT: vpblendvb %xmm2, %xmm4, %xmm3, %xmm2 +; AVX512-NEXT: vpsrlw $8, %xmm2, %xmm2 +; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; AVX512-NEXT: vpsraw $4, %xmm0, %xmm3 +; AVX512-NEXT: vpblendvb %xmm1, %xmm3, %xmm0, %xmm0 +; AVX512-NEXT: vpsraw $2, %xmm0, %xmm3 +; AVX512-NEXT: vpaddw %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm3, %xmm0, %xmm0 +; AVX512-NEXT: vpsraw $1, %xmm0, %xmm3 +; AVX512-NEXT: vpaddw %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm3, %xmm0, %xmm0 +; AVX512-NEXT: vpsrlw $8, %xmm0, %xmm0 +; AVX512-NEXT: vpackuswb %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatvar_shift_v16i8: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3] +; X32-SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7] +; X32-SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm1[0,1,2,3,4,4,4,4] +; X32-SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] +; X32-SSE-NEXT: psllw $5, %xmm3 +; X32-SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15] +; X32-SSE-NEXT: pxor %xmm2, %xmm2 +; X32-SSE-NEXT: pxor %xmm5, %xmm5 +; X32-SSE-NEXT: pcmpgtw %xmm4, %xmm5 +; X32-SSE-NEXT: movdqa %xmm5, %xmm6 +; X32-SSE-NEXT: pandn %xmm1, %xmm6 +; X32-SSE-NEXT: psraw $4, %xmm1 +; X32-SSE-NEXT: pand %xmm5, %xmm1 +; X32-SSE-NEXT: por %xmm6, %xmm1 +; X32-SSE-NEXT: paddw %xmm4, %xmm4 +; X32-SSE-NEXT: pxor %xmm5, %xmm5 +; X32-SSE-NEXT: pcmpgtw %xmm4, %xmm5 +; X32-SSE-NEXT: movdqa %xmm5, %xmm6 +; X32-SSE-NEXT: pandn %xmm1, %xmm6 +; X32-SSE-NEXT: psraw $2, %xmm1 +; X32-SSE-NEXT: pand %xmm5, %xmm1 +; X32-SSE-NEXT: por %xmm6, %xmm1 +; X32-SSE-NEXT: paddw %xmm4, %xmm4 +; X32-SSE-NEXT: pxor %xmm5, %xmm5 +; X32-SSE-NEXT: pcmpgtw %xmm4, %xmm5 +; X32-SSE-NEXT: movdqa %xmm5, %xmm4 +; X32-SSE-NEXT: pandn %xmm1, %xmm4 +; X32-SSE-NEXT: psraw $1, %xmm1 +; X32-SSE-NEXT: pand %xmm5, %xmm1 +; X32-SSE-NEXT: por %xmm4, %xmm1 +; X32-SSE-NEXT: psrlw $8, %xmm1 +; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; X32-SSE-NEXT: pxor %xmm4, %xmm4 +; X32-SSE-NEXT: pcmpgtw %xmm3, %xmm4 +; X32-SSE-NEXT: movdqa %xmm4, %xmm5 +; X32-SSE-NEXT: pandn %xmm0, %xmm5 +; X32-SSE-NEXT: psraw $4, %xmm0 +; X32-SSE-NEXT: pand %xmm4, %xmm0 +; X32-SSE-NEXT: por %xmm5, %xmm0 +; X32-SSE-NEXT: paddw %xmm3, %xmm3 +; X32-SSE-NEXT: pxor %xmm4, %xmm4 +; X32-SSE-NEXT: pcmpgtw %xmm3, %xmm4 +; X32-SSE-NEXT: movdqa %xmm4, %xmm5 +; X32-SSE-NEXT: pandn %xmm0, %xmm5 +; X32-SSE-NEXT: psraw $2, %xmm0 +; X32-SSE-NEXT: pand %xmm4, %xmm0 +; X32-SSE-NEXT: por %xmm5, %xmm0 +; X32-SSE-NEXT: paddw %xmm3, %xmm3 +; X32-SSE-NEXT: pcmpgtw %xmm3, %xmm2 +; X32-SSE-NEXT: movdqa %xmm2, %xmm3 +; X32-SSE-NEXT: pandn %xmm0, %xmm3 +; X32-SSE-NEXT: psraw $1, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: por %xmm3, %xmm0 +; X32-SSE-NEXT: psrlw $8, %xmm0 +; X32-SSE-NEXT: packuswb %xmm1, %xmm0 +; X32-SSE-NEXT: retl %splat = shufflevector <16 x i8> %b, <16 x i8> undef, <16 x i32> zeroinitializer %shift = ashr <16 x i8> %a, %splat ret <16 x i8> %shift @@ -635,46 +1016,83 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; Constant Shifts ; -define <2 x i64> @constant_shift_v2i64(<2 x i64> %a) { +define <2 x i64> @constant_shift_v2i64(<2 x i64> %a) nounwind { ; SSE2-LABEL: constant_shift_v2i64: ; SSE2: # BB#0: -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: sarq %rax -; SSE2-NEXT: movd %rax, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: sarq $7, %rax -; SSE2-NEXT: movd %rax, %xmm0 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: psrlq $7, %xmm1 +; SSE2-NEXT: psrlq $1, %xmm0 +; SSE2-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1] +; SSE2-NEXT: movapd {{.*#+}} xmm0 = [4611686018427387904,72057594037927936] +; SSE2-NEXT: xorpd %xmm0, %xmm1 +; SSE2-NEXT: psubq %xmm0, %xmm1 ; SSE2-NEXT: movdqa %xmm1, %xmm0 ; SSE2-NEXT: retq ; ; SSE41-LABEL: constant_shift_v2i64: ; SSE41: # BB#0: -; SSE41-NEXT: pextrq $1, %xmm0, %rax -; SSE41-NEXT: sarq $7, %rax -; SSE41-NEXT: movd %rax, %xmm1 -; SSE41-NEXT: movd %xmm0, %rax -; SSE41-NEXT: sarq %rax -; SSE41-NEXT: movd %rax, %xmm0 -; SSE41-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: psrlq $7, %xmm1 +; SSE41-NEXT: psrlq $1, %xmm0 +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7] +; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [4611686018427387904,72057594037927936] +; SSE41-NEXT: pxor %xmm1, %xmm0 +; SSE41-NEXT: psubq %xmm1, %xmm0 ; SSE41-NEXT: retq ; -; AVX-LABEL: constant_shift_v2i64: -; AVX: # BB#0: -; AVX-NEXT: vpextrq $1, %xmm0, %rax -; AVX-NEXT: sarq $7, %rax -; AVX-NEXT: vmovq %rax, %xmm1 -; AVX-NEXT: vmovq %xmm0, %rax -; AVX-NEXT: sarq %rax -; AVX-NEXT: vmovq %rax, %xmm0 -; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] -; AVX-NEXT: retq +; AVX1-LABEL: constant_shift_v2i64: +; AVX1: # BB#0: +; AVX1-NEXT: vpsrlq $7, %xmm0, %xmm1 +; AVX1-NEXT: vpsrlq $1, %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7] +; AVX1-NEXT: vmovdqa {{.*#+}} xmm1 = [4611686018427387904,72057594037927936] +; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: constant_shift_v2i64: +; AVX2: # BB#0: +; AVX2-NEXT: vpsrlvq {{.*}}(%rip), %xmm0, %xmm0 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [4611686018427387904,72057594037927936] +; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpsubq %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; XOP-LABEL: constant_shift_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOP-NEXT: vpsubq {{.*}}(%rip), %xmm1, %xmm1 +; XOP-NEXT: vpshaq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: constant_shift_v2i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlvq {{.*}}(%rip), %xmm0, %xmm0 +; AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = [4611686018427387904,72057594037927936] +; AVX512-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpsubq %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: constant_shift_v2i64: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [0,2147483648,0,2147483648] +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: psrlq $7, %xmm2 +; X32-SSE-NEXT: psrlq $1, %xmm1 +; X32-SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1] +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psrlq $7, %xmm1 +; X32-SSE-NEXT: psrlq $1, %xmm0 +; X32-SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1] +; X32-SSE-NEXT: xorpd %xmm2, %xmm1 +; X32-SSE-NEXT: psubq %xmm2, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm0 +; X32-SSE-NEXT: retl %shift = ashr <2 x i64> %a, ret <2 x i64> %shift } -define <4 x i32> @constant_shift_v4i32(<4 x i32> %a) { +define <4 x i32> @constant_shift_v4i32(<4 x i32> %a) nounwind { ; SSE2-LABEL: constant_shift_v4i32: ; SSE2: # BB#0: ; SSE2-NEXT: movdqa %xmm0, %xmm1 @@ -720,11 +1138,42 @@ define <4 x i32> @constant_shift_v4i32(<4 x i32> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsravd {{.*}}(%rip), %xmm0, %xmm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_shift_v4i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpshad {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_shift_v4i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsravd {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: constant_shift_v4i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsravd {{.*}}(%rip), %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: constant_shift_v4i32: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psrad $7, %xmm1 +; X32-SSE-NEXT: movdqa %xmm0, %xmm2 +; X32-SSE-NEXT: psrad $5, %xmm2 +; X32-SSE-NEXT: movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3] +; X32-SSE-NEXT: movdqa %xmm0, %xmm2 +; X32-SSE-NEXT: psrad $6, %xmm2 +; X32-SSE-NEXT: psrad $4, %xmm0 +; X32-SSE-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3] +; X32-SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; X32-SSE-NEXT: retl %shift = ashr <4 x i32> %a, ret <4 x i32> %shift } -define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { +define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) nounwind { ; SSE2-LABEL: constant_shift_v8i16: ; SSE2: # BB#0: ; SSE2-NEXT: movdqa %xmm0, %xmm1 @@ -789,11 +1238,41 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq +; +; XOP-LABEL: constant_shift_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOP-NEXT: vpsubw {{.*}}(%rip), %xmm1, %xmm1 +; XOP-NEXT: vpshaw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: constant_shift_v8i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7] +; AVX512-NEXT: vpsravw %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: constant_shift_v8i16: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psraw $4, %xmm1 +; X32-SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3] +; X32-SSE-NEXT: psraw $2, %xmm1 +; X32-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3] +; X32-SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] +; X32-SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,0,65535,0,65535,0,65535,0] +; X32-SSE-NEXT: movdqa %xmm2, %xmm1 +; X32-SSE-NEXT: pand %xmm0, %xmm1 +; X32-SSE-NEXT: psraw $1, %xmm2 +; X32-SSE-NEXT: pandn %xmm2, %xmm0 +; X32-SSE-NEXT: por %xmm1, %xmm0 +; X32-SSE-NEXT: retl %shift = ashr <8 x i16> %a, ret <8 x i16> %shift } -define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { +define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) nounwind { ; SSE2-LABEL: constant_shift_v16i8: ; SSE2: # BB#0: ; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] @@ -918,6 +1397,101 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX-NEXT: vpsrlw $8, %xmm0, %xmm0 ; AVX-NEXT: vpackuswb %xmm2, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: constant_shift_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOP-NEXT: vpsubb {{.*}}(%rip), %xmm1, %xmm1 +; XOP-NEXT: vpshab %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: constant_shift_v16i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0] +; AVX512-NEXT: vpsllw $5, %xmm1, %xmm1 +; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] +; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] +; AVX512-NEXT: vpsraw $4, %xmm3, %xmm4 +; AVX512-NEXT: vpblendvb %xmm2, %xmm4, %xmm3, %xmm3 +; AVX512-NEXT: vpsraw $2, %xmm3, %xmm4 +; AVX512-NEXT: vpaddw %xmm2, %xmm2, %xmm2 +; AVX512-NEXT: vpblendvb %xmm2, %xmm4, %xmm3, %xmm3 +; AVX512-NEXT: vpsraw $1, %xmm3, %xmm4 +; AVX512-NEXT: vpaddw %xmm2, %xmm2, %xmm2 +; AVX512-NEXT: vpblendvb %xmm2, %xmm4, %xmm3, %xmm2 +; AVX512-NEXT: vpsrlw $8, %xmm2, %xmm2 +; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; AVX512-NEXT: vpsraw $4, %xmm0, %xmm3 +; AVX512-NEXT: vpblendvb %xmm1, %xmm3, %xmm0, %xmm0 +; AVX512-NEXT: vpsraw $2, %xmm0, %xmm3 +; AVX512-NEXT: vpaddw %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm3, %xmm0, %xmm0 +; AVX512-NEXT: vpsraw $1, %xmm0, %xmm3 +; AVX512-NEXT: vpaddw %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm3, %xmm0, %xmm0 +; AVX512-NEXT: vpsrlw $8, %xmm0, %xmm0 +; AVX512-NEXT: vpackuswb %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: constant_shift_v16i8: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15] +; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0] +; X32-SSE-NEXT: psllw $5, %xmm3 +; X32-SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15] +; X32-SSE-NEXT: pxor %xmm2, %xmm2 +; X32-SSE-NEXT: pxor %xmm5, %xmm5 +; X32-SSE-NEXT: pcmpgtw %xmm4, %xmm5 +; X32-SSE-NEXT: movdqa %xmm5, %xmm6 +; X32-SSE-NEXT: pandn %xmm1, %xmm6 +; X32-SSE-NEXT: psraw $4, %xmm1 +; X32-SSE-NEXT: pand %xmm5, %xmm1 +; X32-SSE-NEXT: por %xmm6, %xmm1 +; X32-SSE-NEXT: paddw %xmm4, %xmm4 +; X32-SSE-NEXT: pxor %xmm5, %xmm5 +; X32-SSE-NEXT: pcmpgtw %xmm4, %xmm5 +; X32-SSE-NEXT: movdqa %xmm5, %xmm6 +; X32-SSE-NEXT: pandn %xmm1, %xmm6 +; X32-SSE-NEXT: psraw $2, %xmm1 +; X32-SSE-NEXT: pand %xmm5, %xmm1 +; X32-SSE-NEXT: por %xmm6, %xmm1 +; X32-SSE-NEXT: paddw %xmm4, %xmm4 +; X32-SSE-NEXT: pxor %xmm5, %xmm5 +; X32-SSE-NEXT: pcmpgtw %xmm4, %xmm5 +; X32-SSE-NEXT: movdqa %xmm5, %xmm4 +; X32-SSE-NEXT: pandn %xmm1, %xmm4 +; X32-SSE-NEXT: psraw $1, %xmm1 +; X32-SSE-NEXT: pand %xmm5, %xmm1 +; X32-SSE-NEXT: por %xmm4, %xmm1 +; X32-SSE-NEXT: psrlw $8, %xmm1 +; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; X32-SSE-NEXT: pxor %xmm4, %xmm4 +; X32-SSE-NEXT: pcmpgtw %xmm3, %xmm4 +; X32-SSE-NEXT: movdqa %xmm4, %xmm5 +; X32-SSE-NEXT: pandn %xmm0, %xmm5 +; X32-SSE-NEXT: psraw $4, %xmm0 +; X32-SSE-NEXT: pand %xmm4, %xmm0 +; X32-SSE-NEXT: por %xmm5, %xmm0 +; X32-SSE-NEXT: paddw %xmm3, %xmm3 +; X32-SSE-NEXT: pxor %xmm4, %xmm4 +; X32-SSE-NEXT: pcmpgtw %xmm3, %xmm4 +; X32-SSE-NEXT: movdqa %xmm4, %xmm5 +; X32-SSE-NEXT: pandn %xmm0, %xmm5 +; X32-SSE-NEXT: psraw $2, %xmm0 +; X32-SSE-NEXT: pand %xmm4, %xmm0 +; X32-SSE-NEXT: por %xmm5, %xmm0 +; X32-SSE-NEXT: paddw %xmm3, %xmm3 +; X32-SSE-NEXT: pcmpgtw %xmm3, %xmm2 +; X32-SSE-NEXT: movdqa %xmm2, %xmm3 +; X32-SSE-NEXT: pandn %xmm0, %xmm3 +; X32-SSE-NEXT: psraw $1, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: por %xmm3, %xmm0 +; X32-SSE-NEXT: psrlw $8, %xmm0 +; X32-SSE-NEXT: packuswb %xmm1, %xmm0 +; X32-SSE-NEXT: retl %shift = ashr <16 x i8> %a, ret <16 x i8> %shift } @@ -926,7 +1500,7 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; Uniform Constant Shifts ; -define <2 x i64> @splatconstant_shift_v2i64(<2 x i64> %a) { +define <2 x i64> @splatconstant_shift_v2i64(<2 x i64> %a) nounwind { ; SSE2-LABEL: splatconstant_shift_v2i64: ; SSE2: # BB#0: ; SSE2-NEXT: movdqa %xmm0, %xmm1 @@ -958,11 +1532,35 @@ define <2 x i64> @splatconstant_shift_v2i64(<2 x i64> %a) { ; AVX2-NEXT: vpsrlq $7, %xmm0, %xmm0 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] ; AVX2-NEXT: retq +; +; XOP-LABEL: splatconstant_shift_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOP-NEXT: vpsubq {{.*}}(%rip), %xmm1, %xmm1 +; XOP-NEXT: vpshaq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v2i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrad $7, %xmm0, %xmm1 +; AVX512-NEXT: vpsrlq $7, %xmm0, %xmm0 +; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_shift_v2i64: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psrad $7, %xmm1 +; X32-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3] +; X32-SSE-NEXT: psrlq $7, %xmm0 +; X32-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; X32-SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; X32-SSE-NEXT: retl %shift = ashr <2 x i64> %a, ret <2 x i64> %shift } -define <4 x i32> @splatconstant_shift_v4i32(<4 x i32> %a) { +define <4 x i32> @splatconstant_shift_v4i32(<4 x i32> %a) nounwind { ; SSE-LABEL: splatconstant_shift_v4i32: ; SSE: # BB#0: ; SSE-NEXT: psrad $5, %xmm0 @@ -972,11 +1570,26 @@ define <4 x i32> @splatconstant_shift_v4i32(<4 x i32> %a) { ; AVX: # BB#0: ; AVX-NEXT: vpsrad $5, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_shift_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpsrad $5, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v4i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrad $5, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_shift_v4i32: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: psrad $5, %xmm0 +; X32-SSE-NEXT: retl %shift = ashr <4 x i32> %a, ret <4 x i32> %shift } -define <8 x i16> @splatconstant_shift_v8i16(<8 x i16> %a) { +define <8 x i16> @splatconstant_shift_v8i16(<8 x i16> %a) nounwind { ; SSE-LABEL: splatconstant_shift_v8i16: ; SSE: # BB#0: ; SSE-NEXT: psraw $3, %xmm0 @@ -986,11 +1599,26 @@ define <8 x i16> @splatconstant_shift_v8i16(<8 x i16> %a) { ; AVX: # BB#0: ; AVX-NEXT: vpsraw $3, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_shift_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpsraw $3, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v8i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsraw $3, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_shift_v8i16: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: psraw $3, %xmm0 +; X32-SSE-NEXT: retl %shift = ashr <8 x i16> %a, ret <8 x i16> %shift } -define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { +define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind { ; SSE-LABEL: splatconstant_shift_v16i8: ; SSE: # BB#0: ; SSE-NEXT: psrlw $3, %xmm0 @@ -1008,6 +1636,31 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { ; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 ; AVX-NEXT: vpsubb %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_shift_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOP-NEXT: vpsubb {{.*}}(%rip), %xmm1, %xmm1 +; XOP-NEXT: vpshab %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v16i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlw $3, %xmm0, %xmm0 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] +; AVX512-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpsubb %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_shift_v16i8: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: psrlw $3, %xmm0 +; X32-SSE-NEXT: pand .LCPI15_0, %xmm0 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] +; X32-SSE-NEXT: pxor %xmm1, %xmm0 +; X32-SSE-NEXT: psubb %xmm1, %xmm0 +; X32-SSE-NEXT: retl %shift = ashr <16 x i8> %a, ret <16 x i8> %shift } diff --git a/test/CodeGen/X86/vector-shift-ashr-256.ll b/test/CodeGen/X86/vector-shift-ashr-256.ll index e4642558e0e4..0b9c318da047 100644 --- a/test/CodeGen/X86/vector-shift-ashr-256.ll +++ b/test/CodeGen/X86/vector-shift-ashr-256.ll @@ -1,65 +1,83 @@ -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 - +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX2 +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW ; ; Variable Shifts ; -define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) { +define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind { ; AVX1-LABEL: var_shift_v4i64: ; AVX1: # BB#0: -; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 -; AVX1-NEXT: vpextrq $1, %xmm2, %rax -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3 -; AVX1-NEXT: vpextrq $1, %xmm3, %rcx -; AVX1-NEXT: sarq %cl, %rax -; AVX1-NEXT: vmovq %rax, %xmm4 -; AVX1-NEXT: vmovq %xmm2, %rax -; AVX1-NEXT: vmovq %xmm3, %rcx -; AVX1-NEXT: sarq %cl, %rax -; AVX1-NEXT: vmovq %rax, %xmm2 -; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0] -; AVX1-NEXT: vpextrq $1, %xmm0, %rax -; AVX1-NEXT: vpextrq $1, %xmm1, %rcx -; AVX1-NEXT: sarq %cl, %rax -; AVX1-NEXT: vmovq %rax, %xmm3 -; AVX1-NEXT: vmovq %xmm0, %rax -; AVX1-NEXT: vmovq %xmm1, %rcx -; AVX1-NEXT: sarq %cl, %rax -; AVX1-NEXT: vmovq %rax, %xmm0 -; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0] +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vpsrlq %xmm2, %xmm3, %xmm4 +; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,3,0,1] +; AVX1-NEXT: vpsrlq %xmm5, %xmm3, %xmm6 +; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm6[4,5,6,7] +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6 +; AVX1-NEXT: vpsrlq %xmm2, %xmm6, %xmm2 +; AVX1-NEXT: vpsrlq %xmm5, %xmm6, %xmm5 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm5[4,5,6,7] +; AVX1-NEXT: vpxor %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vpsubq %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vpsrlq %xmm1, %xmm3, %xmm4 +; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,0,1] +; AVX1-NEXT: vpsrlq %xmm5, %xmm3, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7] +; AVX1-NEXT: vpsrlq %xmm1, %xmm0, %xmm1 +; AVX1-NEXT: vpsrlq %xmm5, %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7] +; AVX1-NEXT: vpxor %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpsubq %xmm3, %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: var_shift_v4i64: ; AVX2: # BB#0: -; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2 -; AVX2-NEXT: vpextrq $1, %xmm2, %rax -; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3 -; AVX2-NEXT: vpextrq $1, %xmm3, %rcx -; AVX2-NEXT: sarq %cl, %rax -; AVX2-NEXT: vmovq %rax, %xmm4 -; AVX2-NEXT: vmovq %xmm2, %rax -; AVX2-NEXT: vmovq %xmm3, %rcx -; AVX2-NEXT: sarq %cl, %rax -; AVX2-NEXT: vmovq %rax, %xmm2 -; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0] -; AVX2-NEXT: vpextrq $1, %xmm0, %rax -; AVX2-NEXT: vpextrq $1, %xmm1, %rcx -; AVX2-NEXT: sarq %cl, %rax -; AVX2-NEXT: vmovq %rax, %xmm3 -; AVX2-NEXT: vmovq %xmm0, %rax -; AVX2-NEXT: vmovq %xmm1, %rcx -; AVX2-NEXT: sarq %cl, %rax -; AVX2-NEXT: vmovq %rax, %xmm0 -; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0] -; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpsrlvq %ymm1, %ymm2, %ymm3 +; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpsrlvq %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpsubq %ymm3, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_shift_v4i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX1-NEXT: vpsubq %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; XOPAVX1-NEXT: vpshaq %xmm2, %xmm4, %xmm2 +; XOPAVX1-NEXT: vpsubq %xmm1, %xmm3, %xmm1 +; XOPAVX1-NEXT: vpshaq %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_shift_v4i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; XOPAVX2-NEXT: vpsrlvq %ymm1, %ymm2, %ymm3 +; XOPAVX2-NEXT: vpxor %ymm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: vpsrlvq %ymm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: vpsubq %ymm3, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: var_shift_v4i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX512-NEXT: vpsrlvq %ymm1, %ymm2, %ymm3 +; AVX512-NEXT: vpxor %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: vpsrlvq %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: vpsubq %ymm3, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = ashr <4 x i64> %a, %b ret <4 x i64> %shift } -define <8 x i32> @var_shift_v8i32(<8 x i32> %a, <8 x i32> %b) { +define <8 x i32> @var_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind { ; AVX1-LABEL: var_shift_v8i32: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 @@ -94,11 +112,33 @@ define <8 x i32> @var_shift_v8i32(<8 x i32> %a, <8 x i32> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsravd %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_shift_v8i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX1-NEXT: vpsubd %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; XOPAVX1-NEXT: vpshad %xmm2, %xmm4, %xmm2 +; XOPAVX1-NEXT: vpsubd %xmm1, %xmm3, %xmm1 +; XOPAVX1-NEXT: vpshad %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_shift_v8i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsravd %ymm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: var_shift_v8i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsravd %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = ashr <8 x i32> %a, %b ret <8 x i32> %shift } -define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { +define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind { ; AVX1-LABEL: var_shift_v16i16: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 @@ -147,11 +187,40 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0 ; AVX2-NEXT: vpackusdw %ymm3, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_shift_v16i16: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX1-NEXT: vpsubw %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; XOPAVX1-NEXT: vpshaw %xmm2, %xmm4, %xmm2 +; XOPAVX1-NEXT: vpsubw %xmm1, %xmm3, %xmm1 +; XOPAVX1-NEXT: vpshaw %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_shift_v16i16: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX2-NEXT: vpsubw %xmm2, %xmm3, %xmm2 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm4 +; XOPAVX2-NEXT: vpshaw %xmm2, %xmm4, %xmm2 +; XOPAVX2-NEXT: vpsubw %xmm1, %xmm3, %xmm1 +; XOPAVX2-NEXT: vpshaw %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: var_shift_v16i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsravw %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: retq %shift = ashr <16 x i16> %a, %b ret <16 x i16> %shift } -define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { +define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind { ; AVX1-LABEL: var_shift_v32i8: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 @@ -234,6 +303,58 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0 ; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_shift_v32i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX1-NEXT: vpsubb %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; XOPAVX1-NEXT: vpshab %xmm2, %xmm4, %xmm2 +; XOPAVX1-NEXT: vpsubb %xmm1, %xmm3, %xmm1 +; XOPAVX1-NEXT: vpshab %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_shift_v32i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX2-NEXT: vpsubb %xmm2, %xmm3, %xmm2 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm4 +; XOPAVX2-NEXT: vpshab %xmm2, %xmm4, %xmm2 +; XOPAVX2-NEXT: vpsubb %xmm1, %xmm3, %xmm1 +; XOPAVX2-NEXT: vpshab %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: var_shift_v32i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllw $5, %ymm1, %ymm1 +; AVX512-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] +; AVX512-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] +; AVX512-NEXT: vpsraw $4, %ymm3, %ymm4 +; AVX512-NEXT: vpblendvb %ymm2, %ymm4, %ymm3, %ymm3 +; AVX512-NEXT: vpsraw $2, %ymm3, %ymm4 +; AVX512-NEXT: vpaddw %ymm2, %ymm2, %ymm2 +; AVX512-NEXT: vpblendvb %ymm2, %ymm4, %ymm3, %ymm3 +; AVX512-NEXT: vpsraw $1, %ymm3, %ymm4 +; AVX512-NEXT: vpaddw %ymm2, %ymm2, %ymm2 +; AVX512-NEXT: vpblendvb %ymm2, %ymm4, %ymm3, %ymm2 +; AVX512-NEXT: vpsrlw $8, %ymm2, %ymm2 +; AVX512-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23] +; AVX512-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] +; AVX512-NEXT: vpsraw $4, %ymm0, %ymm3 +; AVX512-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 +; AVX512-NEXT: vpsraw $2, %ymm0, %ymm3 +; AVX512-NEXT: vpaddw %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 +; AVX512-NEXT: vpsraw $1, %ymm0, %ymm3 +; AVX512-NEXT: vpaddw %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 +; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0 +; AVX512-NEXT: vpackuswb %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = ashr <32 x i8> %a, %b ret <32 x i8> %shift } @@ -242,65 +363,64 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; Uniform Variable Shifts ; -define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) { +define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind { ; AVX1-LABEL: splatvar_shift_v4i64: ; AVX1: # BB#0: -; AVX1-NEXT: vmovddup {{.*#+}} xmm1 = xmm1[0,0] -; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 -; AVX1-NEXT: vpextrq $1, %xmm2, %rdx -; AVX1-NEXT: vpextrq $1, %xmm1, %rax -; AVX1-NEXT: movb %al, %cl -; AVX1-NEXT: sarq %cl, %rdx -; AVX1-NEXT: vmovq %rdx, %xmm3 -; AVX1-NEXT: vmovq %xmm2, %rsi -; AVX1-NEXT: vmovq %xmm1, %rdx -; AVX1-NEXT: movb %dl, %cl -; AVX1-NEXT: sarq %cl, %rsi -; AVX1-NEXT: vmovq %rsi, %xmm1 -; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0] -; AVX1-NEXT: vpextrq $1, %xmm0, %rsi -; AVX1-NEXT: movb %al, %cl -; AVX1-NEXT: sarq %cl, %rsi -; AVX1-NEXT: vmovq %rsi, %xmm2 -; AVX1-NEXT: vmovq %xmm0, %rax -; AVX1-NEXT: movb %dl, %cl -; AVX1-NEXT: sarq %cl, %rax -; AVX1-NEXT: vmovq %rax, %xmm0 -; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vpsrlq %xmm1, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpsrlq %xmm1, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm2, %xmm3, %xmm3 +; AVX1-NEXT: vpsubq %xmm2, %xmm3, %xmm3 +; AVX1-NEXT: vpsrlq %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpsubq %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: splatvar_shift_v4i64: ; AVX2: # BB#0: -; AVX2-NEXT: vpbroadcastq %xmm1, %ymm1 -; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2 -; AVX2-NEXT: vpextrq $1, %xmm2, %rax -; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3 -; AVX2-NEXT: vpextrq $1, %xmm3, %rcx -; AVX2-NEXT: sarq %cl, %rax -; AVX2-NEXT: vmovq %rax, %xmm4 -; AVX2-NEXT: vmovq %xmm2, %rax -; AVX2-NEXT: vmovq %xmm3, %rcx -; AVX2-NEXT: sarq %cl, %rax -; AVX2-NEXT: vmovq %rax, %xmm2 -; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0] -; AVX2-NEXT: vpextrq $1, %xmm0, %rax -; AVX2-NEXT: vpextrq $1, %xmm1, %rcx -; AVX2-NEXT: sarq %cl, %rax -; AVX2-NEXT: vmovq %rax, %xmm3 -; AVX2-NEXT: vmovq %xmm0, %rax -; AVX2-NEXT: vmovq %xmm1, %rcx -; AVX2-NEXT: sarq %cl, %rax -; AVX2-NEXT: vmovq %rax, %xmm0 -; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0] -; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpsrlq %xmm1, %ymm2, %ymm2 +; AVX2-NEXT: vpsrlq %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpsubq %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatvar_shift_v4i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vmovddup {{.*#+}} xmm1 = xmm1[0,0] +; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpsubq %xmm1, %xmm2, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpshaq %xmm1, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpshaq %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatvar_shift_v4i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; XOPAVX2-NEXT: vpsrlq %xmm1, %ymm2, %ymm2 +; XOPAVX2-NEXT: vpsrlq %xmm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: vpxor %ymm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: vpsubq %ymm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v4i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX512-NEXT: vpsrlq %xmm1, %ymm2, %ymm2 +; AVX512-NEXT: vpsrlq %xmm1, %ymm0, %ymm0 +; AVX512-NEXT: vpxor %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: vpsubq %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: retq %splat = shufflevector <4 x i64> %b, <4 x i64> undef, <4 x i32> zeroinitializer %shift = ashr <4 x i64> %a, %splat ret <4 x i64> %shift } -define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) { +define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind { ; AVX1-LABEL: splatvar_shift_v8i32: ; AVX1: # BB#0: ; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 @@ -317,12 +437,36 @@ define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) { ; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7] ; AVX2-NEXT: vpsrad %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatvar_shift_v8i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7] +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpsrad %xmm1, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpsrad %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatvar_shift_v8i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7] +; XOPAVX2-NEXT: vpsrad %xmm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v8i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vxorps %xmm2, %xmm2, %xmm2 +; AVX512-NEXT: vmovss %xmm1, %xmm2, %xmm1 +; AVX512-NEXT: vpsrad %xmm1, %ymm0, %ymm0 +; AVX512-NEXT: retq %splat = shufflevector <8 x i32> %b, <8 x i32> undef, <8 x i32> zeroinitializer %shift = ashr <8 x i32> %a, %splat ret <8 x i32> %shift } -define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { +define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind { ; AVX1-LABEL: splatvar_shift_v16i16: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 @@ -341,12 +485,39 @@ define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX2-NEXT: vmovd %eax, %xmm1 ; AVX2-NEXT: vpsraw %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatvar_shift_v16i16: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vmovd %xmm1, %eax +; XOPAVX1-NEXT: movzwl %ax, %eax +; XOPAVX1-NEXT: vmovd %eax, %xmm1 +; XOPAVX1-NEXT: vpsraw %xmm1, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpsraw %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatvar_shift_v16i16: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vmovd %xmm1, %eax +; XOPAVX2-NEXT: movzwl %ax, %eax +; XOPAVX2-NEXT: vmovd %eax, %xmm1 +; XOPAVX2-NEXT: vpsraw %xmm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v16i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vmovd %xmm1, %eax +; AVX512-NEXT: movzwl %ax, %eax +; AVX512-NEXT: vmovd %eax, %xmm1 +; AVX512-NEXT: vpsraw %xmm1, %ymm0, %ymm0 +; AVX512-NEXT: retq %splat = shufflevector <16 x i16> %b, <16 x i16> undef, <16 x i32> zeroinitializer %shift = ashr <16 x i16> %a, %splat ret <16 x i16> %shift } -define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { +define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind { ; AVX1-LABEL: splatvar_shift_v32i8: ; AVX1: # BB#0: ; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 @@ -424,6 +595,59 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0 ; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatvar_shift_v32i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpsubb %xmm1, %xmm2, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpshab %xmm1, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpshab %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatvar_shift_v32i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpbroadcastb %xmm1, %ymm1 +; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX2-NEXT: vpsubb %xmm2, %xmm3, %xmm2 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm4 +; XOPAVX2-NEXT: vpshab %xmm2, %xmm4, %xmm2 +; XOPAVX2-NEXT: vpsubb %xmm1, %xmm3, %xmm1 +; XOPAVX2-NEXT: vpshab %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v32i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpbroadcastb %xmm1, %ymm1 +; AVX512-NEXT: vpsllw $5, %ymm1, %ymm1 +; AVX512-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] +; AVX512-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] +; AVX512-NEXT: vpsraw $4, %ymm3, %ymm4 +; AVX512-NEXT: vpblendvb %ymm2, %ymm4, %ymm3, %ymm3 +; AVX512-NEXT: vpsraw $2, %ymm3, %ymm4 +; AVX512-NEXT: vpaddw %ymm2, %ymm2, %ymm2 +; AVX512-NEXT: vpblendvb %ymm2, %ymm4, %ymm3, %ymm3 +; AVX512-NEXT: vpsraw $1, %ymm3, %ymm4 +; AVX512-NEXT: vpaddw %ymm2, %ymm2, %ymm2 +; AVX512-NEXT: vpblendvb %ymm2, %ymm4, %ymm3, %ymm2 +; AVX512-NEXT: vpsrlw $8, %ymm2, %ymm2 +; AVX512-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23] +; AVX512-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] +; AVX512-NEXT: vpsraw $4, %ymm0, %ymm3 +; AVX512-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 +; AVX512-NEXT: vpsraw $2, %ymm0, %ymm3 +; AVX512-NEXT: vpaddw %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 +; AVX512-NEXT: vpsraw $1, %ymm0, %ymm3 +; AVX512-NEXT: vpaddw %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 +; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0 +; AVX512-NEXT: vpackuswb %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: retq %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer %shift = ashr <32 x i8> %a, %splat ret <32 x i8> %shift @@ -433,51 +657,64 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; Constant Shifts ; -define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) { +define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) nounwind { ; AVX1-LABEL: constant_shift_v4i64: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX1-NEXT: vpextrq $1, %xmm1, %rax -; AVX1-NEXT: sarq $62, %rax -; AVX1-NEXT: vmovq %rax, %xmm2 -; AVX1-NEXT: vmovq %xmm1, %rax -; AVX1-NEXT: sarq $31, %rax -; AVX1-NEXT: vmovq %rax, %xmm1 -; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] -; AVX1-NEXT: vpextrq $1, %xmm0, %rax -; AVX1-NEXT: sarq $7, %rax -; AVX1-NEXT: vmovq %rax, %xmm2 -; AVX1-NEXT: vmovq %xmm0, %rax -; AVX1-NEXT: sarq %rax -; AVX1-NEXT: vmovq %rax, %xmm0 -; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] +; AVX1-NEXT: vpsrlq $62, %xmm1, %xmm2 +; AVX1-NEXT: vpsrlq $31, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [4294967296,2] +; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vpsubq %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vpsrlq $7, %xmm0, %xmm2 +; AVX1-NEXT: vpsrlq $1, %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [4611686018427387904,72057594037927936] +; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpsubq %xmm2, %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: constant_shift_v4i64: ; AVX2: # BB#0: -; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 -; AVX2-NEXT: vpextrq $1, %xmm1, %rax -; AVX2-NEXT: sarq $62, %rax -; AVX2-NEXT: vmovq %rax, %xmm2 -; AVX2-NEXT: vmovq %xmm1, %rax -; AVX2-NEXT: sarq $31, %rax -; AVX2-NEXT: vmovq %rax, %xmm1 -; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] -; AVX2-NEXT: vpextrq $1, %xmm0, %rax -; AVX2-NEXT: sarq $7, %rax -; AVX2-NEXT: vmovq %rax, %xmm2 -; AVX2-NEXT: vmovq %xmm0, %rax -; AVX2-NEXT: sarq %rax -; AVX2-NEXT: vmovq %rax, %xmm0 -; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] -; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: vpsrlvq {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm1 = [4611686018427387904,72057594037927936,4294967296,2] +; AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_shift_v4i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpsubq {{.*}}(%rip), %xmm1, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; XOPAVX1-NEXT: vpshaq %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vpsubq {{.*}}(%rip), %xmm1, %xmm1 +; XOPAVX1-NEXT: vpshaq %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_shift_v4i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsrlvq {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX2-NEXT: vmovdqa {{.*#+}} ymm1 = [4611686018427387904,72057594037927936,4294967296,2] +; XOPAVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: constant_shift_v4i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlvq {{.*}}(%rip), %ymm0, %ymm0 +; AVX512-NEXT: vmovdqa {{.*#+}} ymm1 = [4611686018427387904,72057594037927936,4294967296,2] +; AVX512-NEXT: vpxor %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: vpsubq %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = ashr <4 x i64> %a, ret <4 x i64> %shift } -define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) { +define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) nounwind { ; AVX1-LABEL: constant_shift_v8i32: ; AVX1: # BB#0: ; AVX1-NEXT: vpsrad $7, %xmm0, %xmm1 @@ -500,11 +737,29 @@ define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsravd {{.*}}(%rip), %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_shift_v8i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpshad {{.*}}(%rip), %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vpshad {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_shift_v8i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsravd {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: constant_shift_v8i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsravd {{.*}}(%rip), %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = ashr <8 x i32> %a, ret <8 x i32> %shift } -define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { +define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) nounwind { ; AVX1-LABEL: constant_shift_v16i16: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -549,11 +804,39 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0 ; AVX2-NEXT: vpackusdw %ymm3, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_shift_v16i16: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpsubw {{.*}}(%rip), %xmm1, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; XOPAVX1-NEXT: vpshaw %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vpsubw {{.*}}(%rip), %xmm1, %xmm1 +; XOPAVX1-NEXT: vpshaw %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_shift_v16i16: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOPAVX2-NEXT: vpsubw {{.*}}(%rip), %xmm1, %xmm2 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm3 +; XOPAVX2-NEXT: vpshaw %xmm2, %xmm3, %xmm2 +; XOPAVX2-NEXT: vpsubw {{.*}}(%rip), %xmm1, %xmm1 +; XOPAVX2-NEXT: vpshaw %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: constant_shift_v16i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] +; AVX512-NEXT: vpsravw %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: retq %shift = ashr <16 x i16> %a, ret <16 x i16> %shift } -define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { +define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind { ; AVX1-LABEL: constant_shift_v32i8: ; AVX1: # BB#0: ; AVX1-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0] @@ -630,6 +913,55 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0 ; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_shift_v32i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpsubb {{.*}}(%rip), %xmm1, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpshab %xmm1, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpshab %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_shift_v32i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOPAVX2-NEXT: vpsubb {{.*}}(%rip), %xmm1, %xmm1 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm2 +; XOPAVX2-NEXT: vpshab %xmm1, %xmm2, %xmm2 +; XOPAVX2-NEXT: vpshab %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: constant_shift_v32i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0,0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0] +; AVX512-NEXT: vpsllw $5, %ymm1, %ymm1 +; AVX512-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] +; AVX512-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] +; AVX512-NEXT: vpsraw $4, %ymm3, %ymm4 +; AVX512-NEXT: vpblendvb %ymm2, %ymm4, %ymm3, %ymm3 +; AVX512-NEXT: vpsraw $2, %ymm3, %ymm4 +; AVX512-NEXT: vpaddw %ymm2, %ymm2, %ymm2 +; AVX512-NEXT: vpblendvb %ymm2, %ymm4, %ymm3, %ymm3 +; AVX512-NEXT: vpsraw $1, %ymm3, %ymm4 +; AVX512-NEXT: vpaddw %ymm2, %ymm2, %ymm2 +; AVX512-NEXT: vpblendvb %ymm2, %ymm4, %ymm3, %ymm2 +; AVX512-NEXT: vpsrlw $8, %ymm2, %ymm2 +; AVX512-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23] +; AVX512-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] +; AVX512-NEXT: vpsraw $4, %ymm0, %ymm3 +; AVX512-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 +; AVX512-NEXT: vpsraw $2, %ymm0, %ymm3 +; AVX512-NEXT: vpaddw %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 +; AVX512-NEXT: vpsraw $1, %ymm0, %ymm3 +; AVX512-NEXT: vpaddw %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 +; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0 +; AVX512-NEXT: vpackuswb %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = ashr <32 x i8> %a, ret <32 x i8> %shift } @@ -638,7 +970,7 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; Uniform Constant Shifts ; -define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) { +define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) nounwind { ; AVX1-LABEL: splatconstant_shift_v4i64: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -657,11 +989,36 @@ define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) { ; AVX2-NEXT: vpsrlq $7, %ymm0, %ymm0 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7] ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_shift_v4i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpsubq {{.*}}(%rip), %xmm1, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpshaq %xmm1, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpshaq %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_shift_v4i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsrlq $7, %ymm0, %ymm0 +; XOPAVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm1 +; XOPAVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v4i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrad $7, %ymm0, %ymm1 +; AVX512-NEXT: vpsrlq $7, %ymm0, %ymm0 +; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7] +; AVX512-NEXT: retq %shift = ashr <4 x i64> %a, ret <4 x i64> %shift } -define <8 x i32> @splatconstant_shift_v8i32(<8 x i32> %a) { +define <8 x i32> @splatconstant_shift_v8i32(<8 x i32> %a) nounwind { ; AVX1-LABEL: splatconstant_shift_v8i32: ; AVX1: # BB#0: ; AVX1-NEXT: vpsrad $5, %xmm0, %xmm1 @@ -674,11 +1031,29 @@ define <8 x i32> @splatconstant_shift_v8i32(<8 x i32> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsrad $5, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_shift_v8i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpsrad $5, %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vpsrad $5, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_shift_v8i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsrad $5, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v8i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrad $5, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = ashr <8 x i32> %a, ret <8 x i32> %shift } -define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { +define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) nounwind { ; AVX1-LABEL: splatconstant_shift_v16i16: ; AVX1: # BB#0: ; AVX1-NEXT: vpsraw $3, %xmm0, %xmm1 @@ -691,11 +1066,29 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsraw $3, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_shift_v16i16: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpsraw $3, %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vpsraw $3, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_shift_v16i16: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsraw $3, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v16i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsraw $3, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = ashr <16 x i16> %a, ret <16 x i16> %shift } -define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { +define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind { ; AVX1-LABEL: splatconstant_shift_v32i8: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -720,6 +1113,34 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: vpsubb %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_shift_v32i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpsubb {{.*}}(%rip), %xmm1, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpshab %xmm1, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpshab %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_shift_v32i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsrlw $3, %ymm0, %ymm0 +; XOPAVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX2-NEXT: vmovdqa {{.*#+}} ymm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] +; XOPAVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: vpsubb %ymm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v32i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlw $3, %ymm0, %ymm0 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; AVX512-NEXT: vmovdqa {{.*#+}} ymm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] +; AVX512-NEXT: vpxor %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: vpsubb %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = ashr <32 x i8> %a, ret <32 x i8> %shift } diff --git a/test/CodeGen/X86/vector-shift-ashr-512.ll b/test/CodeGen/X86/vector-shift-ashr-512.ll new file mode 100644 index 000000000000..147e58f4710e --- /dev/null +++ b/test/CodeGen/X86/vector-shift-ashr-512.ll @@ -0,0 +1,378 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512dq | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512DQ +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW +; +; Variable Shifts +; + +define <8 x i64> @var_shift_v8i64(<8 x i64> %a, <8 x i64> %b) nounwind { +; ALL-LABEL: var_shift_v8i64: +; ALL: ## BB#0: +; ALL-NEXT: vpsravq %zmm1, %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = ashr <8 x i64> %a, %b + ret <8 x i64> %shift +} + +define <16 x i32> @var_shift_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind { +; ALL-LABEL: var_shift_v16i32: +; ALL: ## BB#0: +; ALL-NEXT: vpsravd %zmm1, %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = ashr <16 x i32> %a, %b + ret <16 x i32> %shift +} + +define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind { +; AVX512DQ-LABEL: var_shift_v32i16: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpxor %ymm4, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15] +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm6 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15] +; AVX512DQ-NEXT: vpsravd %ymm5, %ymm6, %ymm5 +; AVX512DQ-NEXT: vpsrld $16, %ymm5, %ymm5 +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11] +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11] +; AVX512DQ-NEXT: vpsravd %ymm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrld $16, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpackusdw %ymm5, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm3[4],ymm4[4],ymm3[5],ymm4[5],ymm3[6],ymm4[6],ymm3[7],ymm4[7],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15] +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm1[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15] +; AVX512DQ-NEXT: vpsravd %ymm2, %ymm5, %ymm2 +; AVX512DQ-NEXT: vpsrld $16, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11] +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11] +; AVX512DQ-NEXT: vpsravd %ymm3, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsrld $16, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpackusdw %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq +; +; AVX512BW-LABEL: var_shift_v32i16: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsravw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %shift = ashr <32 x i16> %a, %b + ret <32 x i16> %shift +} + +define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind { +; AVX512DQ-LABEL: var_shift_v64i8: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpsllw $5, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31] +; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] +; AVX512DQ-NEXT: vpsraw $4, %ymm5, %ymm6 +; AVX512DQ-NEXT: vpblendvb %ymm4, %ymm6, %ymm5, %ymm5 +; AVX512DQ-NEXT: vpsraw $2, %ymm5, %ymm6 +; AVX512DQ-NEXT: vpaddw %ymm4, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpblendvb %ymm4, %ymm6, %ymm5, %ymm5 +; AVX512DQ-NEXT: vpsraw $1, %ymm5, %ymm6 +; AVX512DQ-NEXT: vpaddw %ymm4, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpblendvb %ymm4, %ymm6, %ymm5, %ymm4 +; AVX512DQ-NEXT: vpsrlw $8, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23] +; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] +; AVX512DQ-NEXT: vpsraw $4, %ymm0, %ymm5 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm5, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsraw $2, %ymm0, %ymm5 +; AVX512DQ-NEXT: vpaddw %ymm2, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm5, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsraw $1, %ymm0, %ymm5 +; AVX512DQ-NEXT: vpaddw %ymm2, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm5, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrlw $8, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpackuswb %ymm4, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsllw $5, %ymm3, %ymm2 +; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31] +; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] +; AVX512DQ-NEXT: vpsraw $4, %ymm4, %ymm5 +; AVX512DQ-NEXT: vpblendvb %ymm3, %ymm5, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpsraw $2, %ymm4, %ymm5 +; AVX512DQ-NEXT: vpaddw %ymm3, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpblendvb %ymm3, %ymm5, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpsraw $1, %ymm4, %ymm5 +; AVX512DQ-NEXT: vpaddw %ymm3, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpblendvb %ymm3, %ymm5, %ymm4, %ymm3 +; AVX512DQ-NEXT: vpsrlw $8, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23] +; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23] +; AVX512DQ-NEXT: vpsraw $4, %ymm1, %ymm4 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm4, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsraw $2, %ymm1, %ymm4 +; AVX512DQ-NEXT: vpaddw %ymm2, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm4, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsraw $1, %ymm1, %ymm4 +; AVX512DQ-NEXT: vpaddw %ymm2, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm4, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsrlw $8, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpackuswb %ymm3, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq + %shift = ashr <64 x i8> %a, %b + ret <64 x i8> %shift +} + +; +; Uniform Variable Shifts +; + +define <8 x i64> @splatvar_shift_v8i64(<8 x i64> %a, <8 x i64> %b) nounwind { +; ALL-LABEL: splatvar_shift_v8i64: +; ALL: ## BB#0: +; ALL-NEXT: vpsraq %xmm1, %zmm0, %zmm0 +; ALL-NEXT: retq + %splat = shufflevector <8 x i64> %b, <8 x i64> undef, <8 x i32> zeroinitializer + %shift = ashr <8 x i64> %a, %splat + ret <8 x i64> %shift +} + +define <16 x i32> @splatvar_shift_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind { +; ALL-LABEL: splatvar_shift_v16i32: +; ALL: ## BB#0: +; ALL-NEXT: vxorps %xmm2, %xmm2, %xmm2 +; ALL-NEXT: vmovss %xmm1, %xmm2, %xmm1 +; ALL-NEXT: vpsrad %xmm1, %zmm0, %zmm0 +; ALL-NEXT: retq + %splat = shufflevector <16 x i32> %b, <16 x i32> undef, <16 x i32> zeroinitializer + %shift = ashr <16 x i32> %a, %splat + ret <16 x i32> %shift +} + +define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind { +; AVX512DQ-LABEL: splatvar_shift_v32i16: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vmovd %xmm2, %eax +; AVX512DQ-NEXT: movzwl %ax, %eax +; AVX512DQ-NEXT: vmovd %eax, %xmm2 +; AVX512DQ-NEXT: vpsraw %xmm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsraw %xmm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq +; +; AVX512BW-LABEL: splatvar_shift_v32i16: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vmovd %xmm1, %eax +; AVX512BW-NEXT: movzwl %ax, %eax +; AVX512BW-NEXT: vmovd %eax, %xmm1 +; AVX512BW-NEXT: vpsraw %xmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %splat = shufflevector <32 x i16> %b, <32 x i16> undef, <32 x i32> zeroinitializer + %shift = ashr <32 x i16> %a, %splat + ret <32 x i16> %shift +} + +define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind { +; AVX512DQ-LABEL: splatvar_shift_v64i8: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpbroadcastb %xmm2, %ymm2 +; AVX512DQ-NEXT: vpsllw $5, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31] +; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] +; AVX512DQ-NEXT: vpsraw $4, %ymm4, %ymm5 +; AVX512DQ-NEXT: vpblendvb %ymm3, %ymm5, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpsraw $2, %ymm4, %ymm5 +; AVX512DQ-NEXT: vpaddw %ymm3, %ymm3, %ymm6 +; AVX512DQ-NEXT: vpblendvb %ymm6, %ymm5, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpsraw $1, %ymm4, %ymm5 +; AVX512DQ-NEXT: vpaddw %ymm6, %ymm6, %ymm7 +; AVX512DQ-NEXT: vpblendvb %ymm7, %ymm5, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpsrlw $8, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23] +; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] +; AVX512DQ-NEXT: vpsraw $4, %ymm0, %ymm5 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm5, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsraw $2, %ymm0, %ymm5 +; AVX512DQ-NEXT: vpaddw %ymm2, %ymm2, %ymm8 +; AVX512DQ-NEXT: vpblendvb %ymm8, %ymm5, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsraw $1, %ymm0, %ymm5 +; AVX512DQ-NEXT: vpaddw %ymm8, %ymm8, %ymm9 +; AVX512DQ-NEXT: vpblendvb %ymm9, %ymm5, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrlw $8, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpackuswb %ymm4, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] +; AVX512DQ-NEXT: vpsraw $4, %ymm4, %ymm5 +; AVX512DQ-NEXT: vpblendvb %ymm3, %ymm5, %ymm4, %ymm3 +; AVX512DQ-NEXT: vpsraw $2, %ymm3, %ymm4 +; AVX512DQ-NEXT: vpblendvb %ymm6, %ymm4, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpsraw $1, %ymm3, %ymm4 +; AVX512DQ-NEXT: vpblendvb %ymm7, %ymm4, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpsrlw $8, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23] +; AVX512DQ-NEXT: vpsraw $4, %ymm1, %ymm4 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm4, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsraw $2, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm8, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsraw $1, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm9, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsrlw $8, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpackuswb %ymm3, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq + %splat = shufflevector <64 x i8> %b, <64 x i8> undef, <64 x i32> zeroinitializer + %shift = ashr <64 x i8> %a, %splat + ret <64 x i8> %shift +} + +; +; Constant Shifts +; + +define <8 x i64> @constant_shift_v8i64(<8 x i64> %a) nounwind { +; ALL-LABEL: constant_shift_v8i64: +; ALL: ## BB#0: +; ALL-NEXT: vpsravq {{.*}}(%rip), %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = ashr <8 x i64> %a, + ret <8 x i64> %shift +} + +define <16 x i32> @constant_shift_v16i32(<16 x i32> %a) nounwind { +; ALL-LABEL: constant_shift_v16i32: +; ALL: ## BB#0: +; ALL-NEXT: vpsravd {{.*}}(%rip), %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = ashr <16 x i32> %a, + ret <16 x i32> %shift +} + +define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) nounwind { +; AVX512DQ-LABEL: constant_shift_v32i16: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpxor %ymm2, %ymm2, %ymm2 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[12],ymm2[12],ymm3[13],ymm2[13],ymm3[14],ymm2[14],ymm3[15],ymm2[15] +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15] +; AVX512DQ-NEXT: vpsravd %ymm4, %ymm5, %ymm5 +; AVX512DQ-NEXT: vpsrld $16, %ymm5, %ymm5 +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11] +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11] +; AVX512DQ-NEXT: vpsravd %ymm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrld $16, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpackusdw %ymm5, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15] +; AVX512DQ-NEXT: vpsravd %ymm4, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpsrld $16, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11] +; AVX512DQ-NEXT: vpsravd %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsrld $16, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpackusdw %ymm3, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq +; +; AVX512BW-LABEL: constant_shift_v32i16: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsravw {{.*}}(%rip), %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %shift = ashr <32 x i16> %a, + ret <32 x i16> %shift +} + +define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) nounwind { +; AVX512DQ-LABEL: constant_shift_v64i8: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0,0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0] +; AVX512DQ-NEXT: vpsllw $5, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31] +; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] +; AVX512DQ-NEXT: vpsraw $4, %ymm4, %ymm5 +; AVX512DQ-NEXT: vpblendvb %ymm3, %ymm5, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpsraw $2, %ymm4, %ymm5 +; AVX512DQ-NEXT: vpaddw %ymm3, %ymm3, %ymm6 +; AVX512DQ-NEXT: vpblendvb %ymm6, %ymm5, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpsraw $1, %ymm4, %ymm5 +; AVX512DQ-NEXT: vpaddw %ymm6, %ymm6, %ymm7 +; AVX512DQ-NEXT: vpblendvb %ymm7, %ymm5, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpsrlw $8, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23] +; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] +; AVX512DQ-NEXT: vpsraw $4, %ymm0, %ymm5 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm5, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsraw $2, %ymm0, %ymm5 +; AVX512DQ-NEXT: vpaddw %ymm2, %ymm2, %ymm8 +; AVX512DQ-NEXT: vpblendvb %ymm8, %ymm5, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsraw $1, %ymm0, %ymm5 +; AVX512DQ-NEXT: vpaddw %ymm8, %ymm8, %ymm9 +; AVX512DQ-NEXT: vpblendvb %ymm9, %ymm5, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrlw $8, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpackuswb %ymm4, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] +; AVX512DQ-NEXT: vpsraw $4, %ymm4, %ymm5 +; AVX512DQ-NEXT: vpblendvb %ymm3, %ymm5, %ymm4, %ymm3 +; AVX512DQ-NEXT: vpsraw $2, %ymm3, %ymm4 +; AVX512DQ-NEXT: vpblendvb %ymm6, %ymm4, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpsraw $1, %ymm3, %ymm4 +; AVX512DQ-NEXT: vpblendvb %ymm7, %ymm4, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpsrlw $8, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23] +; AVX512DQ-NEXT: vpsraw $4, %ymm1, %ymm4 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm4, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsraw $2, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm8, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsraw $1, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm9, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsrlw $8, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpackuswb %ymm3, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq + %shift = ashr <64 x i8> %a, + ret <64 x i8> %shift +} + +; +; Uniform Constant Shifts +; + +define <8 x i64> @splatconstant_shift_v8i64(<8 x i64> %a) nounwind { +; ALL-LABEL: splatconstant_shift_v8i64: +; ALL: ## BB#0: +; ALL-NEXT: vpsraq $7, %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = ashr <8 x i64> %a, + ret <8 x i64> %shift +} + +define <16 x i32> @splatconstant_shift_v16i32(<16 x i32> %a) nounwind { +; ALL-LABEL: splatconstant_shift_v16i32: +; ALL: ## BB#0: +; ALL-NEXT: vpsrad $5, %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = ashr <16 x i32> %a, + ret <16 x i32> %shift +} + +define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) nounwind { +; AVX512DQ-LABEL: splatconstant_shift_v32i16: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpsraw $3, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsraw $3, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_shift_v32i16: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsraw $3, %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %shift = ashr <32 x i16> %a, + ret <32 x i16> %shift +} + +define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) nounwind { +; AVX512DQ-LABEL: splatconstant_shift_v64i8: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpsrlw $3, %ymm0, %ymm0 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512DQ-NEXT: vpand %ymm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] +; AVX512DQ-NEXT: vpxor %ymm3, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsubb %ymm3, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrlw $3, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpand %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpxor %ymm3, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsubb %ymm3, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_shift_v64i8: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsrlw $3, %zmm0, %zmm0 +; AVX512BW-NEXT: vpandq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512BW-NEXT: vmovdqu8 {{.*#+}} zmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] +; AVX512BW-NEXT: vpxorq %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: vpsubb %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %shift = ashr <64 x i8> %a, + ret <64 x i8> %shift +} diff --git a/test/CodeGen/X86/vector-shift-lshr-128.ll b/test/CodeGen/X86/vector-shift-lshr-128.ll index ca55800e2713..86e54612ae74 100644 --- a/test/CodeGen/X86/vector-shift-lshr-128.ll +++ b/test/CodeGen/X86/vector-shift-lshr-128.ll @@ -1,13 +1,20 @@ -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX2 +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW +; +; Just one 32-bit run to make sure we do reasonable things for i64 shifts. +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=X32-SSE --check-prefix=X32-SSE2 ; ; Variable Shifts ; -define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) { +define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { ; SSE2-LABEL: var_shift_v2i64: ; SSE2: # BB#0: ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1] @@ -39,11 +46,39 @@ define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsrlvq %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_shift_v2i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpsubq %xmm1, %xmm2, %xmm1 +; XOPAVX1-NEXT: vpshlq %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_shift_v2i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsrlvq %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: var_shift_v2i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlvq %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: var_shift_v2i64: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1] +; X32-SSE-NEXT: movdqa %xmm0, %xmm2 +; X32-SSE-NEXT: psrlq %xmm3, %xmm2 +; X32-SSE-NEXT: movq {{.*#+}} xmm1 = xmm1[0],zero +; X32-SSE-NEXT: psrlq %xmm1, %xmm0 +; X32-SSE-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1] +; X32-SSE-NEXT: movapd %xmm2, %xmm0 +; X32-SSE-NEXT: retl %shift = lshr <2 x i64> %a, %b ret <2 x i64> %shift } -define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) { +define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { ; SSE2-LABEL: var_shift_v4i32: ; SSE2: # BB#0: ; SSE2-NEXT: movdqa %xmm1, %xmm2 @@ -109,11 +144,52 @@ define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsrlvd %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_shift_v4i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpsubd %xmm1, %xmm2, %xmm1 +; XOPAVX1-NEXT: vpshld %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_shift_v4i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsrlvd %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: var_shift_v4i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlvd %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: var_shift_v4i32: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: psrldq {{.*#+}} xmm2 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; X32-SSE-NEXT: movdqa %xmm0, %xmm3 +; X32-SSE-NEXT: psrld %xmm2, %xmm3 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: psrlq $32, %xmm2 +; X32-SSE-NEXT: movdqa %xmm0, %xmm4 +; X32-SSE-NEXT: psrld %xmm2, %xmm4 +; X32-SSE-NEXT: movsd {{.*#+}} xmm3 = xmm4[0],xmm3[1] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,3,2,3] +; X32-SSE-NEXT: pxor %xmm3, %xmm3 +; X32-SSE-NEXT: movdqa %xmm1, %xmm4 +; X32-SSE-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3] +; X32-SSE-NEXT: movdqa %xmm0, %xmm5 +; X32-SSE-NEXT: psrld %xmm4, %xmm5 +; X32-SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] +; X32-SSE-NEXT: psrld %xmm1, %xmm0 +; X32-SSE-NEXT: movsd {{.*#+}} xmm5 = xmm0[0],xmm5[1] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3] +; X32-SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; X32-SSE-NEXT: retl %shift = lshr <4 x i32> %a, %b ret <4 x i32> %shift } -define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { +define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { ; SSE2-LABEL: var_shift_v8i16: ; SSE2: # BB#0: ; SSE2-NEXT: psllw $12, %xmm1 @@ -206,11 +282,58 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq +; +; XOP-LABEL: var_shift_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOP-NEXT: vpsubw %xmm1, %xmm2, %xmm1 +; XOP-NEXT: vpshlw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: var_shift_v8i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlvw %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: var_shift_v8i16: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: psllw $12, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: psraw $15, %xmm2 +; X32-SSE-NEXT: movdqa %xmm2, %xmm3 +; X32-SSE-NEXT: pandn %xmm0, %xmm3 +; X32-SSE-NEXT: psrlw $8, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: por %xmm3, %xmm0 +; X32-SSE-NEXT: paddw %xmm1, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: psraw $15, %xmm2 +; X32-SSE-NEXT: movdqa %xmm2, %xmm3 +; X32-SSE-NEXT: pandn %xmm0, %xmm3 +; X32-SSE-NEXT: psrlw $4, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: por %xmm3, %xmm0 +; X32-SSE-NEXT: paddw %xmm1, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: psraw $15, %xmm2 +; X32-SSE-NEXT: movdqa %xmm2, %xmm3 +; X32-SSE-NEXT: pandn %xmm0, %xmm3 +; X32-SSE-NEXT: psrlw $2, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: por %xmm3, %xmm0 +; X32-SSE-NEXT: paddw %xmm1, %xmm1 +; X32-SSE-NEXT: psraw $15, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: pandn %xmm0, %xmm2 +; X32-SSE-NEXT: psrlw $1, %xmm0 +; X32-SSE-NEXT: pand %xmm1, %xmm0 +; X32-SSE-NEXT: por %xmm2, %xmm0 +; X32-SSE-NEXT: retl %shift = lshr <8 x i16> %a, %b ret <8 x i16> %shift } -define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { +define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { ; SSE2-LABEL: var_shift_v16i8: ; SSE2: # BB#0: ; SSE2-NEXT: psllw $5, %xmm1 @@ -281,6 +404,60 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX-NEXT: vpaddb %xmm1, %xmm1, %xmm1 ; AVX-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: var_shift_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOP-NEXT: vpsubb %xmm1, %xmm2, %xmm1 +; XOP-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: var_shift_v16i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllw $5, %xmm1, %xmm1 +; AVX512-NEXT: vpsrlw $4, %xmm0, %xmm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vpsrlw $2, %xmm0, %xmm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX512-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vpsrlw $1, %xmm0, %xmm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX512-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: var_shift_v16i8: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: psllw $5, %xmm1 +; X32-SSE-NEXT: pxor %xmm2, %xmm2 +; X32-SSE-NEXT: pxor %xmm3, %xmm3 +; X32-SSE-NEXT: pcmpgtb %xmm1, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pandn %xmm0, %xmm4 +; X32-SSE-NEXT: psrlw $4, %xmm0 +; X32-SSE-NEXT: pand .LCPI3_0, %xmm0 +; X32-SSE-NEXT: pand %xmm3, %xmm0 +; X32-SSE-NEXT: por %xmm4, %xmm0 +; X32-SSE-NEXT: paddb %xmm1, %xmm1 +; X32-SSE-NEXT: pxor %xmm3, %xmm3 +; X32-SSE-NEXT: pcmpgtb %xmm1, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pandn %xmm0, %xmm4 +; X32-SSE-NEXT: psrlw $2, %xmm0 +; X32-SSE-NEXT: pand .LCPI3_1, %xmm0 +; X32-SSE-NEXT: pand %xmm3, %xmm0 +; X32-SSE-NEXT: por %xmm4, %xmm0 +; X32-SSE-NEXT: paddb %xmm1, %xmm1 +; X32-SSE-NEXT: pcmpgtb %xmm1, %xmm2 +; X32-SSE-NEXT: movdqa %xmm2, %xmm1 +; X32-SSE-NEXT: pandn %xmm0, %xmm1 +; X32-SSE-NEXT: psrlw $1, %xmm0 +; X32-SSE-NEXT: pand .LCPI3_2, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: por %xmm1, %xmm0 +; X32-SSE-NEXT: retl %shift = lshr <16 x i8> %a, %b ret <16 x i8> %shift } @@ -289,7 +466,7 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; Uniform Variable Shifts ; -define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) { +define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { ; SSE-LABEL: splatvar_shift_v2i64: ; SSE: # BB#0: ; SSE-NEXT: psrlq %xmm1, %xmm0 @@ -299,12 +476,28 @@ define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) { ; AVX: # BB#0: ; AVX-NEXT: vpsrlq %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatvar_shift_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpsrlq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v2i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlq %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatvar_shift_v2i64: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movq {{.*#+}} xmm1 = xmm1[0],zero +; X32-SSE-NEXT: psrlq %xmm1, %xmm0 +; X32-SSE-NEXT: retl %splat = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer %shift = lshr <2 x i64> %a, %splat ret <2 x i64> %shift } -define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) { +define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { ; SSE2-LABEL: splatvar_shift_v4i32: ; SSE2: # BB#0: ; SSE2-NEXT: xorps %xmm2, %xmm2 @@ -325,12 +518,33 @@ define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) { ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7] ; AVX-NEXT: vpsrld %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatvar_shift_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7] +; XOP-NEXT: vpsrld %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v4i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vxorps %xmm2, %xmm2, %xmm2 +; AVX512-NEXT: vmovss %xmm1, %xmm2, %xmm1 +; AVX512-NEXT: vpsrld %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatvar_shift_v4i32: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: xorps %xmm2, %xmm2 +; X32-SSE-NEXT: movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3] +; X32-SSE-NEXT: psrld %xmm2, %xmm0 +; X32-SSE-NEXT: retl %splat = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer %shift = lshr <4 x i32> %a, %splat ret <4 x i32> %shift } -define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { +define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { ; SSE2-LABEL: splatvar_shift_v8i16: ; SSE2: # BB#0: ; SSE2-NEXT: movd %xmm1, %eax @@ -352,12 +566,34 @@ define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7] ; AVX-NEXT: vpsrlw %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatvar_shift_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7] +; XOP-NEXT: vpsrlw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v8i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7] +; AVX512-NEXT: vpsrlw %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatvar_shift_v8i16: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movd %xmm1, %eax +; X32-SSE-NEXT: movzwl %ax, %eax +; X32-SSE-NEXT: movd %eax, %xmm1 +; X32-SSE-NEXT: psrlw %xmm1, %xmm0 +; X32-SSE-NEXT: retl %splat = shufflevector <8 x i16> %b, <8 x i16> undef, <8 x i32> zeroinitializer %shift = lshr <8 x i16> %a, %splat ret <8 x i16> %shift } -define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { +define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { ; SSE2-LABEL: splatvar_shift_v16i8: ; SSE2: # BB#0: ; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] @@ -454,6 +690,74 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX2-NEXT: vpaddb %xmm1, %xmm1, %xmm1 ; AVX2-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatvar_shift_v16i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpsubb %xmm1, %xmm2, %xmm1 +; XOPAVX1-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatvar_shift_v16i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpbroadcastb %xmm1, %xmm1 +; XOPAVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX2-NEXT: vpsubb %xmm1, %xmm2, %xmm1 +; XOPAVX2-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v16i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpbroadcastb %xmm1, %xmm1 +; AVX512-NEXT: vpsllw $5, %xmm1, %xmm1 +; AVX512-NEXT: vpsrlw $4, %xmm0, %xmm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vpsrlw $2, %xmm0, %xmm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX512-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vpsrlw $1, %xmm0, %xmm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX512-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatvar_shift_v16i8: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3] +; X32-SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7] +; X32-SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,4,4,4,4] +; X32-SSE-NEXT: psllw $5, %xmm2 +; X32-SSE-NEXT: pxor %xmm1, %xmm1 +; X32-SSE-NEXT: pxor %xmm3, %xmm3 +; X32-SSE-NEXT: pcmpgtb %xmm2, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pandn %xmm0, %xmm4 +; X32-SSE-NEXT: psrlw $4, %xmm0 +; X32-SSE-NEXT: pand .LCPI7_0, %xmm0 +; X32-SSE-NEXT: pand %xmm3, %xmm0 +; X32-SSE-NEXT: por %xmm4, %xmm0 +; X32-SSE-NEXT: paddb %xmm2, %xmm2 +; X32-SSE-NEXT: pxor %xmm3, %xmm3 +; X32-SSE-NEXT: pcmpgtb %xmm2, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pandn %xmm0, %xmm4 +; X32-SSE-NEXT: psrlw $2, %xmm0 +; X32-SSE-NEXT: pand .LCPI7_1, %xmm0 +; X32-SSE-NEXT: pand %xmm3, %xmm0 +; X32-SSE-NEXT: por %xmm4, %xmm0 +; X32-SSE-NEXT: paddb %xmm2, %xmm2 +; X32-SSE-NEXT: pcmpgtb %xmm2, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: pandn %xmm0, %xmm2 +; X32-SSE-NEXT: psrlw $1, %xmm0 +; X32-SSE-NEXT: pand .LCPI7_2, %xmm0 +; X32-SSE-NEXT: pand %xmm1, %xmm0 +; X32-SSE-NEXT: por %xmm2, %xmm0 +; X32-SSE-NEXT: retl %splat = shufflevector <16 x i8> %b, <16 x i8> undef, <16 x i32> zeroinitializer %shift = lshr <16 x i8> %a, %splat ret <16 x i8> %shift @@ -463,7 +767,7 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; Constant Shifts ; -define <2 x i64> @constant_shift_v2i64(<2 x i64> %a) { +define <2 x i64> @constant_shift_v2i64(<2 x i64> %a) nounwind { ; SSE2-LABEL: constant_shift_v2i64: ; SSE2: # BB#0: ; SSE2-NEXT: movdqa %xmm0, %xmm1 @@ -492,61 +796,118 @@ define <2 x i64> @constant_shift_v2i64(<2 x i64> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsrlvq {{.*}}(%rip), %xmm0, %xmm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_shift_v2i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpsubq {{.*}}(%rip), %xmm1, %xmm1 +; XOPAVX1-NEXT: vpshlq %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_shift_v2i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsrlvq {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: constant_shift_v2i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlvq {{.*}}(%rip), %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: constant_shift_v2i64: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psrlq $7, %xmm1 +; X32-SSE-NEXT: psrlq $1, %xmm0 +; X32-SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1] +; X32-SSE-NEXT: movapd %xmm1, %xmm0 +; X32-SSE-NEXT: retl %shift = lshr <2 x i64> %a, ret <2 x i64> %shift } -define <4 x i32> @constant_shift_v4i32(<4 x i32> %a) { +define <4 x i32> @constant_shift_v4i32(<4 x i32> %a) nounwind { ; SSE2-LABEL: constant_shift_v4i32: ; SSE2: # BB#0: -; SSE2-NEXT: movdqa %xmm0, %xmm1 -; SSE2-NEXT: psrld $7, %xmm1 -; SSE2-NEXT: movdqa %xmm0, %xmm2 -; SSE2-NEXT: psrld $5, %xmm2 -; SSE2-NEXT: movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1] -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3] -; SSE2-NEXT: movdqa %xmm0, %xmm2 -; SSE2-NEXT: psrld $6, %xmm2 -; SSE2-NEXT: psrld $4, %xmm0 -; SSE2-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] -; SSE2-NEXT: retq +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: psrld $7, %xmm1 +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: psrld $5, %xmm2 +; SSE2-NEXT: movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1] +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3] +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: psrld $6, %xmm2 +; SSE2-NEXT: psrld $4, %xmm0 +; SSE2-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3] +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE2-NEXT: retq ; ; SSE41-LABEL: constant_shift_v4i32: -; SSE41: # BB#0: -; SSE41-NEXT: movdqa %xmm0, %xmm1 -; SSE41-NEXT: psrld $7, %xmm1 -; SSE41-NEXT: movdqa %xmm0, %xmm2 -; SSE41-NEXT: psrld $5, %xmm2 -; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5,6,7] -; SSE41-NEXT: movdqa %xmm0, %xmm1 -; SSE41-NEXT: psrld $6, %xmm1 -; SSE41-NEXT: psrld $4, %xmm0 -; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7] -; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7] -; SSE41-NEXT: retq +; SSE41: # BB#0: +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: psrld $7, %xmm1 +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: psrld $5, %xmm2 +; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5,6,7] +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: psrld $6, %xmm1 +; SSE41-NEXT: psrld $4, %xmm0 +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7] +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7] +; SSE41-NEXT: retq ; ; AVX1-LABEL: constant_shift_v4i32: -; AVX1: # BB#0: -; AVX1-NEXT: vpsrld $7, %xmm0, %xmm1 -; AVX1-NEXT: vpsrld $5, %xmm0, %xmm2 -; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7] -; AVX1-NEXT: vpsrld $6, %xmm0, %xmm2 -; AVX1-NEXT: vpsrld $4, %xmm0, %xmm0 -; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7] -; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7] -; AVX1-NEXT: retq +; AVX1: # BB#0: +; AVX1-NEXT: vpsrld $7, %xmm0, %xmm1 +; AVX1-NEXT: vpsrld $5, %xmm0, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7] +; AVX1-NEXT: vpsrld $6, %xmm0, %xmm2 +; AVX1-NEXT: vpsrld $4, %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7] +; AVX1-NEXT: retq ; ; AVX2-LABEL: constant_shift_v4i32: ; AVX2: # BB#0: ; AVX2-NEXT: vpsrlvd {{.*}}(%rip), %xmm0, %xmm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_shift_v4i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_shift_v4i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsrlvd {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: constant_shift_v4i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlvd {{.*}}(%rip), %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: constant_shift_v4i32: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psrld $7, %xmm1 +; X32-SSE-NEXT: movdqa %xmm0, %xmm2 +; X32-SSE-NEXT: psrld $5, %xmm2 +; X32-SSE-NEXT: movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3] +; X32-SSE-NEXT: movdqa %xmm0, %xmm2 +; X32-SSE-NEXT: psrld $6, %xmm2 +; X32-SSE-NEXT: psrld $4, %xmm0 +; X32-SSE-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3] +; X32-SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; X32-SSE-NEXT: retl %shift = lshr <4 x i32> %a, ret <4 x i32> %shift } -define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { +define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) nounwind { ; SSE2-LABEL: constant_shift_v8i16: ; SSE2: # BB#0: ; SSE2-NEXT: movdqa %xmm0, %xmm1 @@ -611,11 +972,41 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq +; +; XOP-LABEL: constant_shift_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOP-NEXT: vpsubw {{.*}}(%rip), %xmm1, %xmm1 +; XOP-NEXT: vpshlw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: constant_shift_v8i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7] +; AVX512-NEXT: vpsrlvw %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: constant_shift_v8i16: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psrlw $4, %xmm1 +; X32-SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3] +; X32-SSE-NEXT: psrlw $2, %xmm1 +; X32-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3] +; X32-SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] +; X32-SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,0,65535,0,65535,0,65535,0] +; X32-SSE-NEXT: movdqa %xmm2, %xmm1 +; X32-SSE-NEXT: pand %xmm0, %xmm1 +; X32-SSE-NEXT: psrlw $1, %xmm2 +; X32-SSE-NEXT: pandn %xmm2, %xmm0 +; X32-SSE-NEXT: por %xmm1, %xmm0 +; X32-SSE-NEXT: retl %shift = lshr <8 x i16> %a, ret <8 x i16> %shift } -define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { +define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) nounwind { ; SSE2-LABEL: constant_shift_v16i8: ; SSE2: # BB#0: ; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0] @@ -686,6 +1077,62 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX-NEXT: vpaddb %xmm1, %xmm1, %xmm1 ; AVX-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: constant_shift_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOP-NEXT: vpsubb {{.*}}(%rip), %xmm1, %xmm1 +; XOP-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: constant_shift_v16i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0] +; AVX512-NEXT: vpsllw $5, %xmm1, %xmm1 +; AVX512-NEXT: vpsrlw $4, %xmm0, %xmm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vpsrlw $2, %xmm0, %xmm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX512-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vpsrlw $1, %xmm0, %xmm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX512-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: constant_shift_v16i8: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0] +; X32-SSE-NEXT: psllw $5, %xmm2 +; X32-SSE-NEXT: pxor %xmm1, %xmm1 +; X32-SSE-NEXT: pxor %xmm3, %xmm3 +; X32-SSE-NEXT: pcmpgtb %xmm2, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pandn %xmm0, %xmm4 +; X32-SSE-NEXT: psrlw $4, %xmm0 +; X32-SSE-NEXT: pand .LCPI11_1, %xmm0 +; X32-SSE-NEXT: pand %xmm3, %xmm0 +; X32-SSE-NEXT: por %xmm4, %xmm0 +; X32-SSE-NEXT: paddb %xmm2, %xmm2 +; X32-SSE-NEXT: pxor %xmm3, %xmm3 +; X32-SSE-NEXT: pcmpgtb %xmm2, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pandn %xmm0, %xmm4 +; X32-SSE-NEXT: psrlw $2, %xmm0 +; X32-SSE-NEXT: pand .LCPI11_2, %xmm0 +; X32-SSE-NEXT: pand %xmm3, %xmm0 +; X32-SSE-NEXT: por %xmm4, %xmm0 +; X32-SSE-NEXT: paddb %xmm2, %xmm2 +; X32-SSE-NEXT: pcmpgtb %xmm2, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: pandn %xmm0, %xmm2 +; X32-SSE-NEXT: psrlw $1, %xmm0 +; X32-SSE-NEXT: pand .LCPI11_3, %xmm0 +; X32-SSE-NEXT: pand %xmm1, %xmm0 +; X32-SSE-NEXT: por %xmm2, %xmm0 +; X32-SSE-NEXT: retl %shift = lshr <16 x i8> %a, ret <16 x i8> %shift } @@ -694,7 +1141,7 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; Uniform Constant Shifts ; -define <2 x i64> @splatconstant_shift_v2i64(<2 x i64> %a) { +define <2 x i64> @splatconstant_shift_v2i64(<2 x i64> %a) nounwind { ; SSE-LABEL: splatconstant_shift_v2i64: ; SSE: # BB#0: ; SSE-NEXT: psrlq $7, %xmm0 @@ -704,11 +1151,26 @@ define <2 x i64> @splatconstant_shift_v2i64(<2 x i64> %a) { ; AVX: # BB#0: ; AVX-NEXT: vpsrlq $7, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_shift_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpsrlq $7, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v2i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlq $7, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_shift_v2i64: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: psrlq $7, %xmm0 +; X32-SSE-NEXT: retl %shift = lshr <2 x i64> %a, ret <2 x i64> %shift } -define <4 x i32> @splatconstant_shift_v4i32(<4 x i32> %a) { +define <4 x i32> @splatconstant_shift_v4i32(<4 x i32> %a) nounwind { ; SSE-LABEL: splatconstant_shift_v4i32: ; SSE: # BB#0: ; SSE-NEXT: psrld $5, %xmm0 @@ -718,11 +1180,26 @@ define <4 x i32> @splatconstant_shift_v4i32(<4 x i32> %a) { ; AVX: # BB#0: ; AVX-NEXT: vpsrld $5, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_shift_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpsrld $5, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v4i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrld $5, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_shift_v4i32: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: psrld $5, %xmm0 +; X32-SSE-NEXT: retl %shift = lshr <4 x i32> %a, ret <4 x i32> %shift } -define <8 x i16> @splatconstant_shift_v8i16(<8 x i16> %a) { +define <8 x i16> @splatconstant_shift_v8i16(<8 x i16> %a) nounwind { ; SSE-LABEL: splatconstant_shift_v8i16: ; SSE: # BB#0: ; SSE-NEXT: psrlw $3, %xmm0 @@ -732,11 +1209,26 @@ define <8 x i16> @splatconstant_shift_v8i16(<8 x i16> %a) { ; AVX: # BB#0: ; AVX-NEXT: vpsrlw $3, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_shift_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpsrlw $3, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v8i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlw $3, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_shift_v8i16: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: psrlw $3, %xmm0 +; X32-SSE-NEXT: retl %shift = lshr <8 x i16> %a, ret <8 x i16> %shift } -define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { +define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind { ; SSE-LABEL: splatconstant_shift_v16i8: ; SSE: # BB#0: ; SSE-NEXT: psrlw $3, %xmm0 @@ -748,6 +1240,25 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { ; AVX-NEXT: vpsrlw $3, %xmm0, %xmm0 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_shift_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOP-NEXT: vpsubb {{.*}}(%rip), %xmm1, %xmm1 +; XOP-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v16i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlw $3, %xmm0, %xmm0 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_shift_v16i8: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: psrlw $3, %xmm0 +; X32-SSE-NEXT: pand .LCPI15_0, %xmm0 +; X32-SSE-NEXT: retl %shift = lshr <16 x i8> %a, ret <16 x i8> %shift } diff --git a/test/CodeGen/X86/vector-shift-lshr-256.ll b/test/CodeGen/X86/vector-shift-lshr-256.ll index bb0cceed7720..ecc68cf2e278 100644 --- a/test/CodeGen/X86/vector-shift-lshr-256.ll +++ b/test/CodeGen/X86/vector-shift-lshr-256.ll @@ -1,11 +1,14 @@ -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 - +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX2 +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW ; ; Variable Shifts ; -define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) { +define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind { ; AVX1-LABEL: var_shift_v4i64: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 @@ -25,11 +28,33 @@ define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsrlvq %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_shift_v4i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX1-NEXT: vpsubq %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; XOPAVX1-NEXT: vpshlq %xmm2, %xmm4, %xmm2 +; XOPAVX1-NEXT: vpsubq %xmm1, %xmm3, %xmm1 +; XOPAVX1-NEXT: vpshlq %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_shift_v4i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsrlvq %ymm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: var_shift_v4i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlvq %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = lshr <4 x i64> %a, %b ret <4 x i64> %shift } -define <8 x i32> @var_shift_v8i32(<8 x i32> %a, <8 x i32> %b) { +define <8 x i32> @var_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind { ; AVX1-LABEL: var_shift_v8i32: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 @@ -64,11 +89,33 @@ define <8 x i32> @var_shift_v8i32(<8 x i32> %a, <8 x i32> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsrlvd %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_shift_v8i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX1-NEXT: vpsubd %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; XOPAVX1-NEXT: vpshld %xmm2, %xmm4, %xmm2 +; XOPAVX1-NEXT: vpsubd %xmm1, %xmm3, %xmm1 +; XOPAVX1-NEXT: vpshld %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_shift_v8i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsrlvd %ymm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: var_shift_v8i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlvd %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = lshr <8 x i32> %a, %b ret <8 x i32> %shift } -define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { +define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind { ; AVX1-LABEL: var_shift_v16i16: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 @@ -117,11 +164,40 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0 ; AVX2-NEXT: vpackusdw %ymm3, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_shift_v16i16: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX1-NEXT: vpsubw %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; XOPAVX1-NEXT: vpshlw %xmm2, %xmm4, %xmm2 +; XOPAVX1-NEXT: vpsubw %xmm1, %xmm3, %xmm1 +; XOPAVX1-NEXT: vpshlw %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_shift_v16i16: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX2-NEXT: vpsubw %xmm2, %xmm3, %xmm2 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm4 +; XOPAVX2-NEXT: vpshlw %xmm2, %xmm4, %xmm2 +; XOPAVX2-NEXT: vpsubw %xmm1, %xmm3, %xmm1 +; XOPAVX2-NEXT: vpshlw %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: var_shift_v16i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlvw %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: retq %shift = lshr <16 x i16> %a, %b ret <16 x i16> %shift } -define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { +define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind { ; AVX1-LABEL: var_shift_v32i8: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 @@ -171,6 +247,46 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_shift_v32i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX1-NEXT: vpsubb %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; XOPAVX1-NEXT: vpshlb %xmm2, %xmm4, %xmm2 +; XOPAVX1-NEXT: vpsubb %xmm1, %xmm3, %xmm1 +; XOPAVX1-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_shift_v32i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX2-NEXT: vpsubb %xmm2, %xmm3, %xmm2 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm4 +; XOPAVX2-NEXT: vpshlb %xmm2, %xmm4, %xmm2 +; XOPAVX2-NEXT: vpsubb %xmm1, %xmm3, %xmm1 +; XOPAVX2-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: var_shift_v32i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllw $5, %ymm1, %ymm1 +; AVX512-NEXT: vpsrlw $4, %ymm0, %ymm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: vpsrlw $2, %ymm0, %ymm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX512-NEXT: vpaddb %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: vpsrlw $1, %ymm0, %ymm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX512-NEXT: vpaddb %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = lshr <32 x i8> %a, %b ret <32 x i8> %shift } @@ -179,7 +295,7 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; Uniform Variable Shifts ; -define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) { +define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind { ; AVX1-LABEL: splatvar_shift_v4i64: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 @@ -192,12 +308,30 @@ define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsrlq %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatvar_shift_v4i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpsrlq %xmm1, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpsrlq %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatvar_shift_v4i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsrlq %xmm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v4i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlq %xmm1, %ymm0, %ymm0 +; AVX512-NEXT: retq %splat = shufflevector <4 x i64> %b, <4 x i64> undef, <4 x i32> zeroinitializer %shift = lshr <4 x i64> %a, %splat ret <4 x i64> %shift } -define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) { +define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind { ; AVX1-LABEL: splatvar_shift_v8i32: ; AVX1: # BB#0: ; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 @@ -214,12 +348,36 @@ define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) { ; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7] ; AVX2-NEXT: vpsrld %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatvar_shift_v8i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7] +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpsrld %xmm1, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpsrld %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatvar_shift_v8i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7] +; XOPAVX2-NEXT: vpsrld %xmm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v8i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vxorps %xmm2, %xmm2, %xmm2 +; AVX512-NEXT: vmovss %xmm1, %xmm2, %xmm1 +; AVX512-NEXT: vpsrld %xmm1, %ymm0, %ymm0 +; AVX512-NEXT: retq %splat = shufflevector <8 x i32> %b, <8 x i32> undef, <8 x i32> zeroinitializer %shift = lshr <8 x i32> %a, %splat ret <8 x i32> %shift } -define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { +define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind { ; AVX1-LABEL: splatvar_shift_v16i16: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 @@ -238,12 +396,39 @@ define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX2-NEXT: vmovd %eax, %xmm1 ; AVX2-NEXT: vpsrlw %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatvar_shift_v16i16: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vmovd %xmm1, %eax +; XOPAVX1-NEXT: movzwl %ax, %eax +; XOPAVX1-NEXT: vmovd %eax, %xmm1 +; XOPAVX1-NEXT: vpsrlw %xmm1, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpsrlw %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatvar_shift_v16i16: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vmovd %xmm1, %eax +; XOPAVX2-NEXT: movzwl %ax, %eax +; XOPAVX2-NEXT: vmovd %eax, %xmm1 +; XOPAVX2-NEXT: vpsrlw %xmm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v16i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vmovd %xmm1, %eax +; AVX512-NEXT: movzwl %ax, %eax +; AVX512-NEXT: vmovd %eax, %xmm1 +; AVX512-NEXT: vpsrlw %xmm1, %ymm0, %ymm0 +; AVX512-NEXT: retq %splat = shufflevector <16 x i16> %b, <16 x i16> undef, <16 x i32> zeroinitializer %shift = lshr <16 x i16> %a, %splat ret <16 x i16> %shift } -define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { +define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind { ; AVX1-LABEL: splatvar_shift_v32i8: ; AVX1: # BB#0: ; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 @@ -292,6 +477,47 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatvar_shift_v32i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpsubb %xmm1, %xmm2, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpshlb %xmm1, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatvar_shift_v32i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpbroadcastb %xmm1, %ymm1 +; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; XOPAVX2-NEXT: vpsubb %xmm2, %xmm3, %xmm2 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm4 +; XOPAVX2-NEXT: vpshlb %xmm2, %xmm4, %xmm2 +; XOPAVX2-NEXT: vpsubb %xmm1, %xmm3, %xmm1 +; XOPAVX2-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v32i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpbroadcastb %xmm1, %ymm1 +; AVX512-NEXT: vpsrlw $4, %ymm0, %ymm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX512-NEXT: vpsllw $5, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: vpsrlw $2, %ymm0, %ymm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX512-NEXT: vpaddb %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: vpsrlw $1, %ymm0, %ymm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX512-NEXT: vpaddb %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: retq %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer %shift = lshr <32 x i8> %a, %splat ret <32 x i8> %shift @@ -301,7 +527,7 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; Constant Shifts ; -define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) { +define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) nounwind { ; AVX1-LABEL: constant_shift_v4i64: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -318,11 +544,32 @@ define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsrlvq {{.*}}(%rip), %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_shift_v4i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpsubq {{.*}}(%rip), %xmm1, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; XOPAVX1-NEXT: vpshlq %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vpsubq {{.*}}(%rip), %xmm1, %xmm1 +; XOPAVX1-NEXT: vpshlq %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_shift_v4i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsrlvq {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: constant_shift_v4i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlvq {{.*}}(%rip), %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = lshr <4 x i64> %a, ret <4 x i64> %shift } -define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) { +define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) nounwind { ; AVX1-LABEL: constant_shift_v8i32: ; AVX1: # BB#0: ; AVX1-NEXT: vpsrld $7, %xmm0, %xmm1 @@ -345,11 +592,29 @@ define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsrlvd {{.*}}(%rip), %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_shift_v8i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_shift_v8i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsrlvd {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: constant_shift_v8i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlvd {{.*}}(%rip), %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = lshr <8 x i32> %a, ret <8 x i32> %shift } -define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { +define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) nounwind { ; AVX1-LABEL: constant_shift_v16i16: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -394,11 +659,39 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0 ; AVX2-NEXT: vpackusdw %ymm3, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_shift_v16i16: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpsubw {{.*}}(%rip), %xmm1, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; XOPAVX1-NEXT: vpshlw %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vpsubw {{.*}}(%rip), %xmm1, %xmm1 +; XOPAVX1-NEXT: vpshlw %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_shift_v16i16: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOPAVX2-NEXT: vpsubw {{.*}}(%rip), %xmm1, %xmm2 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm3 +; XOPAVX2-NEXT: vpshlw %xmm2, %xmm3, %xmm2 +; XOPAVX2-NEXT: vpsubw {{.*}}(%rip), %xmm1, %xmm1 +; XOPAVX2-NEXT: vpshlw %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: constant_shift_v16i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] +; AVX512-NEXT: vpsrlvw %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: retq %shift = lshr <16 x i16> %a, ret <16 x i16> %shift } -define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { +define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind { ; AVX1-LABEL: constant_shift_v32i8: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -446,6 +739,43 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_shift_v32i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpsubb {{.*}}(%rip), %xmm1, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpshlb %xmm1, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_shift_v32i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOPAVX2-NEXT: vpsubb {{.*}}(%rip), %xmm1, %xmm1 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm2 +; XOPAVX2-NEXT: vpshlb %xmm1, %xmm2, %xmm2 +; XOPAVX2-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: constant_shift_v32i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0,0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0] +; AVX512-NEXT: vpsllw $5, %ymm1, %ymm1 +; AVX512-NEXT: vpsrlw $4, %ymm0, %ymm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: vpsrlw $2, %ymm0, %ymm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX512-NEXT: vpaddb %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: vpsrlw $1, %ymm0, %ymm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX512-NEXT: vpaddb %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = lshr <32 x i8> %a, ret <32 x i8> %shift } @@ -454,7 +784,7 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; Uniform Constant Shifts ; -define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) { +define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) nounwind { ; AVX1-LABEL: splatconstant_shift_v4i64: ; AVX1: # BB#0: ; AVX1-NEXT: vpsrlq $7, %xmm0, %xmm1 @@ -467,11 +797,29 @@ define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsrlq $7, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_shift_v4i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpsrlq $7, %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vpsrlq $7, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_shift_v4i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsrlq $7, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v4i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlq $7, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = lshr <4 x i64> %a, ret <4 x i64> %shift } -define <8 x i32> @splatconstant_shift_v8i32(<8 x i32> %a) { +define <8 x i32> @splatconstant_shift_v8i32(<8 x i32> %a) nounwind { ; AVX1-LABEL: splatconstant_shift_v8i32: ; AVX1: # BB#0: ; AVX1-NEXT: vpsrld $5, %xmm0, %xmm1 @@ -484,11 +832,29 @@ define <8 x i32> @splatconstant_shift_v8i32(<8 x i32> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsrld $5, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_shift_v8i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpsrld $5, %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vpsrld $5, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_shift_v8i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsrld $5, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v8i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrld $5, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = lshr <8 x i32> %a, ret <8 x i32> %shift } -define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { +define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) nounwind { ; AVX1-LABEL: splatconstant_shift_v16i16: ; AVX1: # BB#0: ; AVX1-NEXT: vpsrlw $3, %xmm0, %xmm1 @@ -501,11 +867,29 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsrlw $3, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_shift_v16i16: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpsrlw $3, %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vpsrlw $3, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_shift_v16i16: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsrlw $3, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v16i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlw $3, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = lshr <16 x i16> %a, ret <16 x i16> %shift } -define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { +define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind { ; AVX1-LABEL: splatconstant_shift_v32i8: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -522,6 +906,28 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX2-NEXT: vpsrlw $3, %ymm0, %ymm0 ; AVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_shift_v32i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpsubb {{.*}}(%rip), %xmm1, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpshlb %xmm1, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_shift_v32i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsrlw $3, %ymm0, %ymm0 +; XOPAVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v32i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsrlw $3, %ymm0, %ymm0 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = lshr <32 x i8> %a, ret <32 x i8> %shift } diff --git a/test/CodeGen/X86/vector-shift-lshr-512.ll b/test/CodeGen/X86/vector-shift-lshr-512.ll new file mode 100644 index 000000000000..68644e61b0e5 --- /dev/null +++ b/test/CodeGen/X86/vector-shift-lshr-512.ll @@ -0,0 +1,317 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512dq | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512DQ +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW + +; +; Variable Shifts +; + +define <8 x i64> @var_shift_v8i64(<8 x i64> %a, <8 x i64> %b) nounwind { +; ALL-LABEL: var_shift_v8i64: +; ALL: ## BB#0: +; ALL-NEXT: vpsrlvq %zmm1, %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = lshr <8 x i64> %a, %b + ret <8 x i64> %shift +} + +define <16 x i32> @var_shift_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind { +; ALL-LABEL: var_shift_v16i32: +; ALL: ## BB#0: +; ALL-NEXT: vpsrlvd %zmm1, %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = lshr <16 x i32> %a, %b + ret <16 x i32> %shift +} + +define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind { +; AVX512DQ-LABEL: var_shift_v32i16: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpxor %ymm4, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15] +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm6 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15] +; AVX512DQ-NEXT: vpsrlvd %ymm5, %ymm6, %ymm5 +; AVX512DQ-NEXT: vpsrld $16, %ymm5, %ymm5 +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11] +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11] +; AVX512DQ-NEXT: vpsrlvd %ymm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrld $16, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpackusdw %ymm5, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm3[4],ymm4[4],ymm3[5],ymm4[5],ymm3[6],ymm4[6],ymm3[7],ymm4[7],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15] +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm1[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15] +; AVX512DQ-NEXT: vpsrlvd %ymm2, %ymm5, %ymm2 +; AVX512DQ-NEXT: vpsrld $16, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11] +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11] +; AVX512DQ-NEXT: vpsrlvd %ymm3, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsrld $16, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpackusdw %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq +; +; AVX512BW-LABEL: var_shift_v32i16: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsrlvw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %shift = lshr <32 x i16> %a, %b + ret <32 x i16> %shift +} + +define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind { +; AVX512DQ-LABEL: var_shift_v64i8: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpsrlw $4, %ymm0, %ymm4 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX512DQ-NEXT: vpand %ymm5, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpsllw $5, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm4, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrlw $2, %ymm0, %ymm4 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63] +; AVX512DQ-NEXT: vpand %ymm6, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpaddb %ymm2, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm4, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrlw $1, %ymm0, %ymm4 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127] +; AVX512DQ-NEXT: vpand %ymm7, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpaddb %ymm2, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm4, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrlw $4, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpand %ymm5, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpsllw $5, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpblendvb %ymm3, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsrlw $2, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpand %ymm6, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpaddb %ymm3, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpblendvb %ymm3, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsrlw $1, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpand %ymm7, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpaddb %ymm3, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpblendvb %ymm3, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq + + %shift = lshr <64 x i8> %a, %b + ret <64 x i8> %shift +} + +; +; Uniform Variable Shifts +; + +define <8 x i64> @splatvar_shift_v8i64(<8 x i64> %a, <8 x i64> %b) nounwind { +; ALL-LABEL: splatvar_shift_v8i64: +; ALL: ## BB#0: +; ALL-NEXT: vpsrlq %xmm1, %zmm0, %zmm0 +; ALL-NEXT: retq + %splat = shufflevector <8 x i64> %b, <8 x i64> undef, <8 x i32> zeroinitializer + %shift = lshr <8 x i64> %a, %splat + ret <8 x i64> %shift +} + +define <16 x i32> @splatvar_shift_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind { +; ALL-LABEL: splatvar_shift_v16i32: +; ALL: ## BB#0: +; ALL-NEXT: vxorps %xmm2, %xmm2, %xmm2 +; ALL-NEXT: vmovss %xmm1, %xmm2, %xmm1 +; ALL-NEXT: vpsrld %xmm1, %zmm0, %zmm0 +; ALL-NEXT: retq + %splat = shufflevector <16 x i32> %b, <16 x i32> undef, <16 x i32> zeroinitializer + %shift = lshr <16 x i32> %a, %splat + ret <16 x i32> %shift +} + +define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind { +; AVX512DQ-LABEL: splatvar_shift_v32i16: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vmovd %xmm2, %eax +; AVX512DQ-NEXT: movzwl %ax, %eax +; AVX512DQ-NEXT: vmovd %eax, %xmm2 +; AVX512DQ-NEXT: vpsrlw %xmm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrlw %xmm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq +; +; AVX512BW-LABEL: splatvar_shift_v32i16: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vmovd %xmm1, %eax +; AVX512BW-NEXT: movzwl %ax, %eax +; AVX512BW-NEXT: vmovd %eax, %xmm1 +; AVX512BW-NEXT: vpsrlw %xmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %splat = shufflevector <32 x i16> %b, <32 x i16> undef, <32 x i32> zeroinitializer + %shift = lshr <32 x i16> %a, %splat + ret <32 x i16> %shift +} + +define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind { +; AVX512DQ-LABEL: splatvar_shift_v64i8: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpbroadcastb %xmm2, %ymm2 +; AVX512DQ-NEXT: vpsrlw $4, %ymm0, %ymm3 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX512DQ-NEXT: vpand %ymm4, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpsllw $5, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm3, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrlw $2, %ymm0, %ymm3 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm5 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63] +; AVX512DQ-NEXT: vpand %ymm5, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpaddb %ymm2, %ymm2, %ymm6 +; AVX512DQ-NEXT: vpblendvb %ymm6, %ymm3, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrlw $1, %ymm0, %ymm3 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127] +; AVX512DQ-NEXT: vpand %ymm7, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpaddb %ymm6, %ymm6, %ymm8 +; AVX512DQ-NEXT: vpblendvb %ymm8, %ymm3, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrlw $4, %ymm1, %ymm3 +; AVX512DQ-NEXT: vpand %ymm4, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm3, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsrlw $2, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpand %ymm5, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm6, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsrlw $1, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpand %ymm7, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm8, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq + %splat = shufflevector <64 x i8> %b, <64 x i8> undef, <64 x i32> zeroinitializer + %shift = lshr <64 x i8> %a, %splat + ret <64 x i8> %shift +} + +; +; Constant Shifts +; + +define <8 x i64> @constant_shift_v8i64(<8 x i64> %a) nounwind { +; ALL-LABEL: constant_shift_v8i64: +; ALL: ## BB#0: +; ALL-NEXT: vpsrlvq {{.*}}(%rip), %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = lshr <8 x i64> %a, + ret <8 x i64> %shift +} + +define <16 x i32> @constant_shift_v16i32(<16 x i32> %a) nounwind { +; ALL-LABEL: constant_shift_v16i32: +; ALL: ## BB#0: +; ALL-NEXT: vpsrlvd {{.*}}(%rip), %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = lshr <16 x i32> %a, + ret <16 x i32> %shift +} + +define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) nounwind { +; AVX512DQ-LABEL: constant_shift_v32i16: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpxor %ymm2, %ymm2, %ymm2 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[12],ymm2[12],ymm3[13],ymm2[13],ymm3[14],ymm2[14],ymm3[15],ymm2[15] +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15] +; AVX512DQ-NEXT: vpsrlvd %ymm4, %ymm5, %ymm5 +; AVX512DQ-NEXT: vpsrld $16, %ymm5, %ymm5 +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11] +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11] +; AVX512DQ-NEXT: vpsrlvd %ymm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrld $16, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpackusdw %ymm5, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15] +; AVX512DQ-NEXT: vpsrlvd %ymm4, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpsrld $16, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11] +; AVX512DQ-NEXT: vpsrlvd %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsrld $16, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpackusdw %ymm3, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq +; +; AVX512BW-LABEL: constant_shift_v32i16: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsrlvw {{.*}}(%rip), %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %shift = lshr <32 x i16> %a, + ret <32 x i16> %shift +} + +define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) nounwind { +; AVX512DQ-LABEL: constant_shift_v64i8: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpsrlw $4, %ymm0, %ymm2 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX512DQ-NEXT: vpand %ymm3, %ymm2, %ymm2 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0,0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0] +; AVX512DQ-NEXT: vpsllw $5, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpblendvb %ymm4, %ymm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrlw $2, %ymm0, %ymm2 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm5 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63] +; AVX512DQ-NEXT: vpand %ymm5, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpaddb %ymm4, %ymm4, %ymm6 +; AVX512DQ-NEXT: vpblendvb %ymm6, %ymm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrlw $1, %ymm0, %ymm2 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127] +; AVX512DQ-NEXT: vpand %ymm7, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpaddb %ymm6, %ymm6, %ymm8 +; AVX512DQ-NEXT: vpblendvb %ymm8, %ymm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrlw $4, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpand %ymm3, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm4, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsrlw $2, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpand %ymm5, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm6, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsrlw $1, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpand %ymm7, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm8, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq + %shift = lshr <64 x i8> %a, + ret <64 x i8> %shift +} + +; +; Uniform Constant Shifts +; + +define <8 x i64> @splatconstant_shift_v8i64(<8 x i64> %a) nounwind { +; ALL-LABEL: splatconstant_shift_v8i64: +; ALL: ## BB#0: +; ALL-NEXT: vpsrlq $7, %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = lshr <8 x i64> %a, + ret <8 x i64> %shift +} + +define <16 x i32> @splatconstant_shift_v16i32(<16 x i32> %a) nounwind { +; ALL-LABEL: splatconstant_shift_v16i32: +; ALL: ## BB#0: +; ALL-NEXT: vpsrld $5, %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = lshr <16 x i32> %a, + ret <16 x i32> %shift +} + +define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) nounwind { +; AVX512DQ-LABEL: splatconstant_shift_v32i16: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpsrlw $3, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrlw $3, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_shift_v32i16: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsrlw $3, %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %shift = lshr <32 x i16> %a, + ret <32 x i16> %shift +} + +define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) nounwind { +; AVX512DQ-LABEL: splatconstant_shift_v64i8: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpsrlw $3, %ymm0, %ymm0 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31] +; AVX512DQ-NEXT: vpand %ymm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrlw $3, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpand %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_shift_v64i8: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsrlw $3, %zmm0, %zmm0 +; AVX512BW-NEXT: vpandq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %shift = lshr <64 x i8> %a, + ret <64 x i8> %shift +} diff --git a/test/CodeGen/X86/vector-shift-shl-128.ll b/test/CodeGen/X86/vector-shift-shl-128.ll index 6dbd9eab2a72..9b59c6224ef2 100644 --- a/test/CodeGen/X86/vector-shift-shl-128.ll +++ b/test/CodeGen/X86/vector-shift-shl-128.ll @@ -1,13 +1,20 @@ -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX2 +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW +; +; Just one 32-bit run to make sure we do reasonable things for i64 shifts. +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=X32-SSE --check-prefix=X32-SSE2 ; ; Variable Shifts ; -define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) { +define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { ; SSE2-LABEL: var_shift_v2i64: ; SSE2: # BB#0: ; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1] @@ -39,11 +46,37 @@ define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsllvq %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_shift_v2i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpshlq %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_shift_v2i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsllvq %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: var_shift_v2i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllvq %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: var_shift_v2i64: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1] +; X32-SSE-NEXT: movdqa %xmm0, %xmm2 +; X32-SSE-NEXT: psllq %xmm3, %xmm2 +; X32-SSE-NEXT: movq {{.*#+}} xmm1 = xmm1[0],zero +; X32-SSE-NEXT: psllq %xmm1, %xmm0 +; X32-SSE-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1] +; X32-SSE-NEXT: movapd %xmm2, %xmm0 +; X32-SSE-NEXT: retl %shift = shl <2 x i64> %a, %b ret <2 x i64> %shift } -define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) { +define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { ; SSE2-LABEL: var_shift_v4i32: ; SSE2: # BB#0: ; SSE2-NEXT: pslld $23, %xmm1 @@ -79,11 +112,41 @@ define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsllvd %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_shift_v4i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpshld %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_shift_v4i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsllvd %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: var_shift_v4i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllvd %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: var_shift_v4i32: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: pslld $23, %xmm1 +; X32-SSE-NEXT: paddd .LCPI1_0, %xmm1 +; X32-SSE-NEXT: cvttps2dq %xmm1, %xmm1 +; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3] +; X32-SSE-NEXT: pmuludq %xmm0, %xmm1 +; X32-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; X32-SSE-NEXT: pmuludq %xmm2, %xmm0 +; X32-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; X32-SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; X32-SSE-NEXT: movdqa %xmm1, %xmm0 +; X32-SSE-NEXT: retl %shift = shl <4 x i32> %a, %b ret <4 x i32> %shift } -define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { +define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { ; SSE2-LABEL: var_shift_v8i16: ; SSE2: # BB#0: ; SSE2-NEXT: psllw $12, %xmm1 @@ -176,11 +239,56 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq +; +; XOP-LABEL: var_shift_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpshlw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: var_shift_v8i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllvw %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: var_shift_v8i16: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: psllw $12, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: psraw $15, %xmm2 +; X32-SSE-NEXT: movdqa %xmm2, %xmm3 +; X32-SSE-NEXT: pandn %xmm0, %xmm3 +; X32-SSE-NEXT: psllw $8, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: por %xmm3, %xmm0 +; X32-SSE-NEXT: paddw %xmm1, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: psraw $15, %xmm2 +; X32-SSE-NEXT: movdqa %xmm2, %xmm3 +; X32-SSE-NEXT: pandn %xmm0, %xmm3 +; X32-SSE-NEXT: psllw $4, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: por %xmm3, %xmm0 +; X32-SSE-NEXT: paddw %xmm1, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: psraw $15, %xmm2 +; X32-SSE-NEXT: movdqa %xmm2, %xmm3 +; X32-SSE-NEXT: pandn %xmm0, %xmm3 +; X32-SSE-NEXT: psllw $2, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: por %xmm3, %xmm0 +; X32-SSE-NEXT: paddw %xmm1, %xmm1 +; X32-SSE-NEXT: psraw $15, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: pandn %xmm0, %xmm2 +; X32-SSE-NEXT: psllw $1, %xmm0 +; X32-SSE-NEXT: pand %xmm1, %xmm0 +; X32-SSE-NEXT: por %xmm2, %xmm0 +; X32-SSE-NEXT: retl %shift = shl <8 x i16> %a, %b ret <8 x i16> %shift } -define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { +define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { ; SSE2-LABEL: var_shift_v16i8: ; SSE2: # BB#0: ; SSE2-NEXT: psllw $5, %xmm1 @@ -248,6 +356,56 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX-NEXT: vpaddb %xmm1, %xmm1, %xmm1 ; AVX-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: var_shift_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: var_shift_v16i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllw $5, %xmm1, %xmm1 +; AVX512-NEXT: vpsllw $4, %xmm0, %xmm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vpsllw $2, %xmm0, %xmm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX512-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vpaddb %xmm0, %xmm0, %xmm2 +; AVX512-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: var_shift_v16i8: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: psllw $5, %xmm1 +; X32-SSE-NEXT: pxor %xmm2, %xmm2 +; X32-SSE-NEXT: pxor %xmm3, %xmm3 +; X32-SSE-NEXT: pcmpgtb %xmm1, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pandn %xmm0, %xmm4 +; X32-SSE-NEXT: psllw $4, %xmm0 +; X32-SSE-NEXT: pand .LCPI3_0, %xmm0 +; X32-SSE-NEXT: pand %xmm3, %xmm0 +; X32-SSE-NEXT: por %xmm4, %xmm0 +; X32-SSE-NEXT: paddb %xmm1, %xmm1 +; X32-SSE-NEXT: pxor %xmm3, %xmm3 +; X32-SSE-NEXT: pcmpgtb %xmm1, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pandn %xmm0, %xmm4 +; X32-SSE-NEXT: psllw $2, %xmm0 +; X32-SSE-NEXT: pand .LCPI3_1, %xmm0 +; X32-SSE-NEXT: pand %xmm3, %xmm0 +; X32-SSE-NEXT: por %xmm4, %xmm0 +; X32-SSE-NEXT: paddb %xmm1, %xmm1 +; X32-SSE-NEXT: pcmpgtb %xmm1, %xmm2 +; X32-SSE-NEXT: movdqa %xmm2, %xmm1 +; X32-SSE-NEXT: pandn %xmm0, %xmm1 +; X32-SSE-NEXT: paddb %xmm0, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: por %xmm1, %xmm0 +; X32-SSE-NEXT: retl %shift = shl <16 x i8> %a, %b ret <16 x i8> %shift } @@ -256,7 +414,7 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; Uniform Variable Shifts ; -define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) { +define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind { ; SSE-LABEL: splatvar_shift_v2i64: ; SSE: # BB#0: ; SSE-NEXT: psllq %xmm1, %xmm0 @@ -266,12 +424,28 @@ define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) { ; AVX: # BB#0: ; AVX-NEXT: vpsllq %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatvar_shift_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpsllq %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v2i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllq %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatvar_shift_v2i64: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movq {{.*#+}} xmm1 = xmm1[0],zero +; X32-SSE-NEXT: psllq %xmm1, %xmm0 +; X32-SSE-NEXT: retl %splat = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer %shift = shl <2 x i64> %a, %splat ret <2 x i64> %shift } -define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) { +define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind { ; SSE2-LABEL: splatvar_shift_v4i32: ; SSE2: # BB#0: ; SSE2-NEXT: xorps %xmm2, %xmm2 @@ -292,12 +466,33 @@ define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) { ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7] ; AVX-NEXT: vpslld %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatvar_shift_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7] +; XOP-NEXT: vpslld %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v4i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vxorps %xmm2, %xmm2, %xmm2 +; AVX512-NEXT: vmovss %xmm1, %xmm2, %xmm1 +; AVX512-NEXT: vpslld %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatvar_shift_v4i32: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: xorps %xmm2, %xmm2 +; X32-SSE-NEXT: movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3] +; X32-SSE-NEXT: pslld %xmm2, %xmm0 +; X32-SSE-NEXT: retl %splat = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer %shift = shl <4 x i32> %a, %splat ret <4 x i32> %shift } -define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { +define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind { ; SSE2-LABEL: splatvar_shift_v8i16: ; SSE2: # BB#0: ; SSE2-NEXT: movd %xmm1, %eax @@ -319,12 +514,34 @@ define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7] ; AVX-NEXT: vpsllw %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatvar_shift_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7] +; XOP-NEXT: vpsllw %xmm1, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v8i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7] +; AVX512-NEXT: vpsllw %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatvar_shift_v8i16: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movd %xmm1, %eax +; X32-SSE-NEXT: movzwl %ax, %eax +; X32-SSE-NEXT: movd %eax, %xmm1 +; X32-SSE-NEXT: psllw %xmm1, %xmm0 +; X32-SSE-NEXT: retl %splat = shufflevector <8 x i16> %b, <8 x i16> undef, <8 x i32> zeroinitializer %shift = shl <8 x i16> %a, %splat ret <8 x i16> %shift } -define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { +define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind { ; SSE2-LABEL: splatvar_shift_v16i8: ; SSE2: # BB#0: ; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] @@ -417,6 +634,69 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX2-NEXT: vpaddb %xmm1, %xmm1, %xmm1 ; AVX2-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatvar_shift_v16i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatvar_shift_v16i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpbroadcastb %xmm1, %xmm1 +; XOPAVX2-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v16i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpbroadcastb %xmm1, %xmm1 +; AVX512-NEXT: vpsllw $5, %xmm1, %xmm1 +; AVX512-NEXT: vpsllw $4, %xmm0, %xmm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vpsllw $2, %xmm0, %xmm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX512-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vpaddb %xmm0, %xmm0, %xmm2 +; AVX512-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatvar_shift_v16i8: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3] +; X32-SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7] +; X32-SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,4,4,4,4] +; X32-SSE-NEXT: psllw $5, %xmm2 +; X32-SSE-NEXT: pxor %xmm1, %xmm1 +; X32-SSE-NEXT: pxor %xmm3, %xmm3 +; X32-SSE-NEXT: pcmpgtb %xmm2, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pandn %xmm0, %xmm4 +; X32-SSE-NEXT: psllw $4, %xmm0 +; X32-SSE-NEXT: pand .LCPI7_0, %xmm0 +; X32-SSE-NEXT: pand %xmm3, %xmm0 +; X32-SSE-NEXT: por %xmm4, %xmm0 +; X32-SSE-NEXT: paddb %xmm2, %xmm2 +; X32-SSE-NEXT: pxor %xmm3, %xmm3 +; X32-SSE-NEXT: pcmpgtb %xmm2, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pandn %xmm0, %xmm4 +; X32-SSE-NEXT: psllw $2, %xmm0 +; X32-SSE-NEXT: pand .LCPI7_1, %xmm0 +; X32-SSE-NEXT: pand %xmm3, %xmm0 +; X32-SSE-NEXT: por %xmm4, %xmm0 +; X32-SSE-NEXT: paddb %xmm2, %xmm2 +; X32-SSE-NEXT: pcmpgtb %xmm2, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: pandn %xmm0, %xmm2 +; X32-SSE-NEXT: paddb %xmm0, %xmm0 +; X32-SSE-NEXT: pand %xmm1, %xmm0 +; X32-SSE-NEXT: por %xmm2, %xmm0 +; X32-SSE-NEXT: retl %splat = shufflevector <16 x i8> %b, <16 x i8> undef, <16 x i32> zeroinitializer %shift = shl <16 x i8> %a, %splat ret <16 x i8> %shift @@ -426,7 +706,7 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; Constant Shifts ; -define <2 x i64> @constant_shift_v2i64(<2 x i64> %a) { +define <2 x i64> @constant_shift_v2i64(<2 x i64> %a) nounwind { ; SSE2-LABEL: constant_shift_v2i64: ; SSE2: # BB#0: ; SSE2-NEXT: movdqa %xmm0, %xmm1 @@ -455,11 +735,35 @@ define <2 x i64> @constant_shift_v2i64(<2 x i64> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_shift_v2i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpshlq {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_shift_v2i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: constant_shift_v2i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: constant_shift_v2i64: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psllq $7, %xmm1 +; X32-SSE-NEXT: psllq $1, %xmm0 +; X32-SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1] +; X32-SSE-NEXT: movapd %xmm1, %xmm0 +; X32-SSE-NEXT: retl %shift = shl <2 x i64> %a, ret <2 x i64> %shift } -define <4 x i32> @constant_shift_v4i32(<4 x i32> %a) { +define <4 x i32> @constant_shift_v4i32(<4 x i32> %a) nounwind { ; SSE2-LABEL: constant_shift_v4i32: ; SSE2: # BB#0: ; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [16,32,64,128] @@ -486,11 +790,38 @@ define <4 x i32> @constant_shift_v4i32(<4 x i32> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_shift_v4i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_shift_v4i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: constant_shift_v4i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: constant_shift_v4i32: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [16,32,64,128] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3] +; X32-SSE-NEXT: pmuludq %xmm1, %xmm0 +; X32-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; X32-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; X32-SSE-NEXT: pmuludq %xmm2, %xmm1 +; X32-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; X32-SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; X32-SSE-NEXT: retl %shift = shl <4 x i32> %a, ret <4 x i32> %shift } -define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { +define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) nounwind { ; SSE-LABEL: constant_shift_v8i16: ; SSE: # BB#0: ; SSE-NEXT: pmullw {{.*}}(%rip), %xmm0 @@ -500,11 +831,27 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX: # BB#0: ; AVX-NEXT: vpmullw {{.*}}(%rip), %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: constant_shift_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpshlw {{.*}}(%rip), %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: constant_shift_v8i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7] +; AVX512-NEXT: vpsllvw %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: constant_shift_v8i16: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: pmullw .LCPI10_0, %xmm0 +; X32-SSE-NEXT: retl %shift = shl <8 x i16> %a, ret <8 x i16> %shift } -define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { +define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) nounwind { ; SSE2-LABEL: constant_shift_v16i8: ; SSE2: # BB#0: ; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0] @@ -572,6 +919,58 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX-NEXT: vpaddb %xmm1, %xmm1, %xmm1 ; AVX-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: constant_shift_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpshlb {{.*}}(%rip), %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: constant_shift_v16i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0] +; AVX512-NEXT: vpsllw $5, %xmm1, %xmm1 +; AVX512-NEXT: vpsllw $4, %xmm0, %xmm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vpsllw $2, %xmm0, %xmm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm2, %xmm2 +; AVX512-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vpaddb %xmm0, %xmm0, %xmm2 +; AVX512-NEXT: vpaddb %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: constant_shift_v16i8: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0] +; X32-SSE-NEXT: psllw $5, %xmm2 +; X32-SSE-NEXT: pxor %xmm1, %xmm1 +; X32-SSE-NEXT: pxor %xmm3, %xmm3 +; X32-SSE-NEXT: pcmpgtb %xmm2, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pandn %xmm0, %xmm4 +; X32-SSE-NEXT: psllw $4, %xmm0 +; X32-SSE-NEXT: pand .LCPI11_1, %xmm0 +; X32-SSE-NEXT: pand %xmm3, %xmm0 +; X32-SSE-NEXT: por %xmm4, %xmm0 +; X32-SSE-NEXT: paddb %xmm2, %xmm2 +; X32-SSE-NEXT: pxor %xmm3, %xmm3 +; X32-SSE-NEXT: pcmpgtb %xmm2, %xmm3 +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pandn %xmm0, %xmm4 +; X32-SSE-NEXT: psllw $2, %xmm0 +; X32-SSE-NEXT: pand .LCPI11_2, %xmm0 +; X32-SSE-NEXT: pand %xmm3, %xmm0 +; X32-SSE-NEXT: por %xmm4, %xmm0 +; X32-SSE-NEXT: paddb %xmm2, %xmm2 +; X32-SSE-NEXT: pcmpgtb %xmm2, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm2 +; X32-SSE-NEXT: pandn %xmm0, %xmm2 +; X32-SSE-NEXT: paddb %xmm0, %xmm0 +; X32-SSE-NEXT: pand %xmm1, %xmm0 +; X32-SSE-NEXT: por %xmm2, %xmm0 +; X32-SSE-NEXT: retl %shift = shl <16 x i8> %a, ret <16 x i8> %shift } @@ -580,7 +979,7 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; Uniform Constant Shifts ; -define <2 x i64> @splatconstant_shift_v2i64(<2 x i64> %a) { +define <2 x i64> @splatconstant_shift_v2i64(<2 x i64> %a) nounwind { ; SSE-LABEL: splatconstant_shift_v2i64: ; SSE: # BB#0: ; SSE-NEXT: psllq $7, %xmm0 @@ -590,11 +989,26 @@ define <2 x i64> @splatconstant_shift_v2i64(<2 x i64> %a) { ; AVX: # BB#0: ; AVX-NEXT: vpsllq $7, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_shift_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vpsllq $7, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v2i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllq $7, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_shift_v2i64: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: psllq $7, %xmm0 +; X32-SSE-NEXT: retl %shift = shl <2 x i64> %a, ret <2 x i64> %shift } -define <4 x i32> @splatconstant_shift_v4i32(<4 x i32> %a) { +define <4 x i32> @splatconstant_shift_v4i32(<4 x i32> %a) nounwind { ; SSE-LABEL: splatconstant_shift_v4i32: ; SSE: # BB#0: ; SSE-NEXT: pslld $5, %xmm0 @@ -604,11 +1018,26 @@ define <4 x i32> @splatconstant_shift_v4i32(<4 x i32> %a) { ; AVX: # BB#0: ; AVX-NEXT: vpslld $5, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_shift_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vpslld $5, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v4i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpslld $5, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_shift_v4i32: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: pslld $5, %xmm0 +; X32-SSE-NEXT: retl %shift = shl <4 x i32> %a, ret <4 x i32> %shift } -define <8 x i16> @splatconstant_shift_v8i16(<8 x i16> %a) { +define <8 x i16> @splatconstant_shift_v8i16(<8 x i16> %a) nounwind { ; SSE-LABEL: splatconstant_shift_v8i16: ; SSE: # BB#0: ; SSE-NEXT: psllw $3, %xmm0 @@ -618,11 +1047,26 @@ define <8 x i16> @splatconstant_shift_v8i16(<8 x i16> %a) { ; AVX: # BB#0: ; AVX-NEXT: vpsllw $3, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_shift_v8i16: +; XOP: # BB#0: +; XOP-NEXT: vpsllw $3, %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v8i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllw $3, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_shift_v8i16: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: psllw $3, %xmm0 +; X32-SSE-NEXT: retl %shift = shl <8 x i16> %a, ret <8 x i16> %shift } -define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { +define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind { ; SSE-LABEL: splatconstant_shift_v16i8: ; SSE: # BB#0: ; SSE-NEXT: psllw $3, %xmm0 @@ -634,6 +1078,23 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { ; AVX-NEXT: vpsllw $3, %xmm0, %xmm0 ; AVX-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 ; AVX-NEXT: retq +; +; XOP-LABEL: splatconstant_shift_v16i8: +; XOP: # BB#0: +; XOP-NEXT: vpshlb {{.*}}(%rip), %xmm0, %xmm0 +; XOP-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v16i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllw $3, %xmm0, %xmm0 +; AVX512-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; X32-SSE-LABEL: splatconstant_shift_v16i8: +; X32-SSE: # BB#0: +; X32-SSE-NEXT: psllw $3, %xmm0 +; X32-SSE-NEXT: pand .LCPI15_0, %xmm0 +; X32-SSE-NEXT: retl %shift = shl <16 x i8> %a, ret <16 x i8> %shift } diff --git a/test/CodeGen/X86/vector-shift-shl-256.ll b/test/CodeGen/X86/vector-shift-shl-256.ll index b287875f6541..3daf24f1a82e 100644 --- a/test/CodeGen/X86/vector-shift-shl-256.ll +++ b/test/CodeGen/X86/vector-shift-shl-256.ll @@ -1,11 +1,15 @@ -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=XOP --check-prefix=XOPAVX2 +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW ; ; Variable Shifts ; -define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) { +define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind { ; AVX1-LABEL: var_shift_v4i64: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 @@ -25,11 +29,30 @@ define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsllvq %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_shift_v4i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; XOPAVX1-NEXT: vpshlq %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vpshlq %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_shift_v4i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsllvq %ymm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: var_shift_v4i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllvq %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = shl <4 x i64> %a, %b ret <4 x i64> %shift } -define <8 x i32> @var_shift_v8i32(<8 x i32> %a, <8 x i32> %b) { +define <8 x i32> @var_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind { ; AVX1-LABEL: var_shift_v8i32: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 @@ -50,11 +73,30 @@ define <8 x i32> @var_shift_v8i32(<8 x i32> %a, <8 x i32> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsllvd %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_shift_v8i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; XOPAVX1-NEXT: vpshld %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vpshld %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_shift_v8i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsllvd %ymm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: var_shift_v8i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllvd %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = shl <8 x i32> %a, %b ret <8 x i32> %shift } -define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { +define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind { ; AVX1-LABEL: var_shift_v16i16: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 @@ -103,11 +145,34 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0 ; AVX2-NEXT: vpackusdw %ymm3, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_shift_v16i16: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; XOPAVX1-NEXT: vpshlw %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vpshlw %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_shift_v16i16: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm3 +; XOPAVX2-NEXT: vpshlw %xmm2, %xmm3, %xmm2 +; XOPAVX2-NEXT: vpshlw %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: var_shift_v16i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllvw %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: retq %shift = shl <16 x i16> %a, %b ret <16 x i16> %shift } -define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { +define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind { ; AVX1-LABEL: var_shift_v32i8: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 @@ -153,6 +218,39 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: var_shift_v32i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; XOPAVX1-NEXT: vpshlb %xmm2, %xmm3, %xmm2 +; XOPAVX1-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: var_shift_v32i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm3 +; XOPAVX2-NEXT: vpshlb %xmm2, %xmm3, %xmm2 +; XOPAVX2-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: var_shift_v32i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllw $5, %ymm1, %ymm1 +; AVX512-NEXT: vpsllw $4, %ymm0, %ymm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: vpsllw $2, %ymm0, %ymm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX512-NEXT: vpaddb %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: vpaddb %ymm0, %ymm0, %ymm2 +; AVX512-NEXT: vpaddb %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = shl <32 x i8> %a, %b ret <32 x i8> %shift } @@ -161,7 +259,7 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; Uniform Variable Shifts ; -define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) { +define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind { ; AVX1-LABEL: splatvar_shift_v4i64: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 @@ -174,12 +272,30 @@ define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsllq %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatvar_shift_v4i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpsllq %xmm1, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpsllq %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatvar_shift_v4i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsllq %xmm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v4i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllq %xmm1, %ymm0, %ymm0 +; AVX512-NEXT: retq %splat = shufflevector <4 x i64> %b, <4 x i64> undef, <4 x i32> zeroinitializer %shift = shl <4 x i64> %a, %splat ret <4 x i64> %shift } -define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) { +define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind { ; AVX1-LABEL: splatvar_shift_v8i32: ; AVX1: # BB#0: ; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 @@ -196,12 +312,36 @@ define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) { ; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7] ; AVX2-NEXT: vpslld %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatvar_shift_v8i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7] +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpslld %xmm1, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpslld %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatvar_shift_v8i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7] +; XOPAVX2-NEXT: vpslld %xmm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v8i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vxorps %xmm2, %xmm2, %xmm2 +; AVX512-NEXT: vmovss %xmm1, %xmm2, %xmm1 +; AVX512-NEXT: vpslld %xmm1, %ymm0, %ymm0 +; AVX512-NEXT: retq %splat = shufflevector <8 x i32> %b, <8 x i32> undef, <8 x i32> zeroinitializer %shift = shl <8 x i32> %a, %splat ret <8 x i32> %shift } -define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { +define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind { ; AVX1-LABEL: splatvar_shift_v16i16: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 @@ -220,12 +360,39 @@ define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX2-NEXT: vmovd %eax, %xmm1 ; AVX2-NEXT: vpsllw %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatvar_shift_v16i16: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vmovd %xmm1, %eax +; XOPAVX1-NEXT: movzwl %ax, %eax +; XOPAVX1-NEXT: vmovd %eax, %xmm1 +; XOPAVX1-NEXT: vpsllw %xmm1, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpsllw %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatvar_shift_v16i16: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vmovd %xmm1, %eax +; XOPAVX2-NEXT: movzwl %ax, %eax +; XOPAVX2-NEXT: vmovd %eax, %xmm1 +; XOPAVX2-NEXT: vpsllw %xmm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v16i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vmovd %xmm1, %eax +; AVX512-NEXT: movzwl %ax, %eax +; AVX512-NEXT: vmovd %eax, %xmm1 +; AVX512-NEXT: vpsllw %xmm1, %ymm0, %ymm0 +; AVX512-NEXT: retq %splat = shufflevector <16 x i16> %b, <16 x i16> undef, <16 x i32> zeroinitializer %shift = shl <16 x i16> %a, %splat ret <16 x i16> %shift } -define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { +define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind { ; AVX1-LABEL: splatvar_shift_v32i8: ; AVX1: # BB#0: ; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 @@ -270,6 +437,42 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatvar_shift_v32i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; XOPAVX1-NEXT: vpshlb %xmm1, %xmm2, %xmm2 +; XOPAVX1-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatvar_shift_v32i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpbroadcastb %xmm1, %ymm1 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm2 +; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm3 +; XOPAVX2-NEXT: vpshlb %xmm3, %xmm2, %xmm2 +; XOPAVX2-NEXT: vpshlb %xmm1, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatvar_shift_v32i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpbroadcastb %xmm1, %ymm1 +; AVX512-NEXT: vpsllw $4, %ymm0, %ymm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX512-NEXT: vpsllw $5, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: vpsllw $2, %ymm0, %ymm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX512-NEXT: vpaddb %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: vpaddb %ymm0, %ymm0, %ymm2 +; AVX512-NEXT: vpaddb %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: retq %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer %shift = shl <32 x i8> %a, %splat ret <32 x i8> %shift @@ -279,7 +482,7 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; Constant Shifts ; -define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) { +define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) nounwind { ; AVX1-LABEL: constant_shift_v4i64: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -296,11 +499,29 @@ define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsllvq {{.*}}(%rip), %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_shift_v4i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpshlq {{.*}}(%rip), %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vpshlq {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_shift_v4i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsllvq {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: constant_shift_v4i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllvq {{.*}}(%rip), %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = shl <4 x i64> %a, ret <4 x i64> %shift } -define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) { +define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) nounwind { ; AVX1-LABEL: constant_shift_v8i32: ; AVX1: # BB#0: ; AVX1-NEXT: vpmulld {{.*}}(%rip), %xmm0, %xmm1 @@ -313,11 +534,29 @@ define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_shift_v8i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_shift_v8i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: constant_shift_v8i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = shl <8 x i32> %a, ret <8 x i32> %shift } -define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { +define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) nounwind { ; AVX1-LABEL: constant_shift_v16i16: ; AVX1: # BB#0: ; AVX1-NEXT: vpmullw {{.*}}(%rip), %xmm0, %xmm1 @@ -330,11 +569,30 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpmullw {{.*}}(%rip), %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_shift_v16i16: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpshlw {{.*}}(%rip), %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vpshlw {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_shift_v16i16: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpmullw {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: constant_shift_v16i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] +; AVX512-NEXT: vpsllvw %zmm1, %zmm0, %zmm0 +; AVX512-NEXT: retq %shift = shl <16 x i16> %a, ret <16 x i16> %shift } -define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { +define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind { ; AVX1-LABEL: constant_shift_v32i8: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -378,6 +636,40 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: constant_shift_v32i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0] +; XOPAVX1-NEXT: vpshlb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpshlb %xmm2, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: constant_shift_v32i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; XOPAVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0] +; XOPAVX2-NEXT: vpshlb %xmm2, %xmm1, %xmm1 +; XOPAVX2-NEXT: vpshlb %xmm2, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: constant_shift_v32i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0,0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0] +; AVX512-NEXT: vpsllw $5, %ymm1, %ymm1 +; AVX512-NEXT: vpsllw $4, %ymm0, %ymm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: vpsllw $2, %ymm0, %ymm2 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm2, %ymm2 +; AVX512-NEXT: vpaddb %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: vpaddb %ymm0, %ymm0, %ymm2 +; AVX512-NEXT: vpaddb %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = shl <32 x i8> %a, ret <32 x i8> %shift } @@ -386,7 +678,7 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; Uniform Constant Shifts ; -define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) { +define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) nounwind { ; AVX1-LABEL: splatconstant_shift_v4i64: ; AVX1: # BB#0: ; AVX1-NEXT: vpsllq $7, %xmm0, %xmm1 @@ -399,11 +691,29 @@ define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsllq $7, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_shift_v4i64: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpsllq $7, %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vpsllq $7, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_shift_v4i64: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsllq $7, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v4i64: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllq $7, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = shl <4 x i64> %a, ret <4 x i64> %shift } -define <8 x i32> @splatconstant_shift_v8i32(<8 x i32> %a) { +define <8 x i32> @splatconstant_shift_v8i32(<8 x i32> %a) nounwind { ; AVX1-LABEL: splatconstant_shift_v8i32: ; AVX1: # BB#0: ; AVX1-NEXT: vpslld $5, %xmm0, %xmm1 @@ -416,11 +726,29 @@ define <8 x i32> @splatconstant_shift_v8i32(<8 x i32> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpslld $5, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_shift_v8i32: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpslld $5, %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vpslld $5, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_shift_v8i32: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpslld $5, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v8i32: +; AVX512: ## BB#0: +; AVX512-NEXT: vpslld $5, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = shl <8 x i32> %a, ret <8 x i32> %shift } -define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { +define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) nounwind { ; AVX1-LABEL: splatconstant_shift_v16i16: ; AVX1: # BB#0: ; AVX1-NEXT: vpsllw $3, %xmm0, %xmm1 @@ -433,11 +761,29 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsllw $3, %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_shift_v16i16: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vpsllw $3, %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vpsllw $3, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_shift_v16i16: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsllw $3, %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v16i16: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllw $3, %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = shl <16 x i16> %a, ret <16 x i16> %shift } -define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { +define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind { ; AVX1-LABEL: splatconstant_shift_v32i8: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 @@ -454,6 +800,27 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX2-NEXT: vpsllw $3, %ymm0, %ymm0 ; AVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 ; AVX2-NEXT: retq +; +; XOPAVX1-LABEL: splatconstant_shift_v32i8: +; XOPAVX1: # BB#0: +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3] +; XOPAVX1-NEXT: vpshlb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpshlb %xmm2, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; XOPAVX1-NEXT: retq +; +; XOPAVX2-LABEL: splatconstant_shift_v32i8: +; XOPAVX2: # BB#0: +; XOPAVX2-NEXT: vpsllw $3, %ymm0, %ymm0 +; XOPAVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; XOPAVX2-NEXT: retq +; +; AVX512-LABEL: splatconstant_shift_v32i8: +; AVX512: ## BB#0: +; AVX512-NEXT: vpsllw $3, %ymm0, %ymm0 +; AVX512-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 +; AVX512-NEXT: retq %shift = shl <32 x i8> %a, ret <32 x i8> %shift } diff --git a/test/CodeGen/X86/vector-shift-shl-512.ll b/test/CodeGen/X86/vector-shift-shl-512.ll new file mode 100644 index 000000000000..26ddb1c127e1 --- /dev/null +++ b/test/CodeGen/X86/vector-shift-shl-512.ll @@ -0,0 +1,293 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512dq | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512DQ +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW + +; +; Variable Shifts +; + +define <8 x i64> @var_shift_v8i64(<8 x i64> %a, <8 x i64> %b) nounwind { +; ALL-LABEL: var_shift_v8i64: +; ALL: ## BB#0: +; ALL-NEXT: vpsllvq %zmm1, %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = shl <8 x i64> %a, %b + ret <8 x i64> %shift +} + +define <16 x i32> @var_shift_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind { +; ALL-LABEL: var_shift_v16i32: +; ALL: ## BB#0: +; ALL-NEXT: vpsllvd %zmm1, %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = shl <16 x i32> %a, %b + ret <16 x i32> %shift +} + +define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind { +; AVX512DQ-LABEL: var_shift_v32i16: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpxor %ymm4, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15] +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm6 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15] +; AVX512DQ-NEXT: vpsllvd %ymm5, %ymm6, %ymm5 +; AVX512DQ-NEXT: vpsrld $16, %ymm5, %ymm5 +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11] +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11] +; AVX512DQ-NEXT: vpsllvd %ymm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsrld $16, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpackusdw %ymm5, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm3[4],ymm4[4],ymm3[5],ymm4[5],ymm3[6],ymm4[6],ymm3[7],ymm4[7],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15] +; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm1[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15] +; AVX512DQ-NEXT: vpsllvd %ymm2, %ymm5, %ymm2 +; AVX512DQ-NEXT: vpsrld $16, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11] +; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11] +; AVX512DQ-NEXT: vpsllvd %ymm3, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsrld $16, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpackusdw %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq +; +; AVX512BW-LABEL: var_shift_v32i16: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsllvw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %shift = shl <32 x i16> %a, %b + ret <32 x i16> %shift +} + +define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind { +; AVX512DQ-LABEL: var_shift_v64i8: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpsllw $4, %ymm0, %ymm4 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240] +; AVX512DQ-NEXT: vpand %ymm5, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpsllw $5, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm4, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsllw $2, %ymm0, %ymm4 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252] +; AVX512DQ-NEXT: vpand %ymm6, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpaddb %ymm2, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm4, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpaddb %ymm0, %ymm0, %ymm4 +; AVX512DQ-NEXT: vpaddb %ymm2, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm4, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsllw $4, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpand %ymm5, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpsllw $5, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpblendvb %ymm3, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsllw $2, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpand %ymm6, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpaddb %ymm3, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpblendvb %ymm3, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpaddb %ymm1, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpaddb %ymm3, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpblendvb %ymm3, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq + %shift = shl <64 x i8> %a, %b + ret <64 x i8> %shift +} + +; +; Uniform Variable Shifts +; + +define <8 x i64> @splatvar_shift_v8i64(<8 x i64> %a, <8 x i64> %b) nounwind { +; ALL-LABEL: splatvar_shift_v8i64: +; ALL: ## BB#0: +; ALL-NEXT: vpsllq %xmm1, %zmm0, %zmm0 +; ALL-NEXT: retq + %splat = shufflevector <8 x i64> %b, <8 x i64> undef, <8 x i32> zeroinitializer + %shift = shl <8 x i64> %a, %splat + ret <8 x i64> %shift +} + +define <16 x i32> @splatvar_shift_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind { +; ALL-LABEL: splatvar_shift_v16i32: +; ALL: ## BB#0: +; ALL-NEXT: vxorps %xmm2, %xmm2, %xmm2 +; ALL-NEXT: vmovss %xmm1, %xmm2, %xmm1 +; ALL-NEXT: vpslld %xmm1, %zmm0, %zmm0 +; ALL-NEXT: retq + %splat = shufflevector <16 x i32> %b, <16 x i32> undef, <16 x i32> zeroinitializer + %shift = shl <16 x i32> %a, %splat + ret <16 x i32> %shift +} + +define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind { +; AVX512DQ-LABEL: splatvar_shift_v32i16: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vmovd %xmm2, %eax +; AVX512DQ-NEXT: movzwl %ax, %eax +; AVX512DQ-NEXT: vmovd %eax, %xmm2 +; AVX512DQ-NEXT: vpsllw %xmm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsllw %xmm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq +; +; AVX512BW-LABEL: splatvar_shift_v32i16: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vmovd %xmm1, %eax +; AVX512BW-NEXT: movzwl %ax, %eax +; AVX512BW-NEXT: vmovd %eax, %xmm1 +; AVX512BW-NEXT: vpsllw %xmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %splat = shufflevector <32 x i16> %b, <32 x i16> undef, <32 x i32> zeroinitializer + %shift = shl <32 x i16> %a, %splat + ret <32 x i16> %shift +} + +define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind { +; AVX512DQ-LABEL: splatvar_shift_v64i8: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpbroadcastb %xmm2, %ymm2 +; AVX512DQ-NEXT: vpsllw $4, %ymm0, %ymm3 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240] +; AVX512DQ-NEXT: vpand %ymm4, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpsllw $5, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm3, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsllw $2, %ymm0, %ymm3 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm5 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252] +; AVX512DQ-NEXT: vpand %ymm5, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpaddb %ymm2, %ymm2, %ymm6 +; AVX512DQ-NEXT: vpblendvb %ymm6, %ymm3, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpaddb %ymm0, %ymm0, %ymm3 +; AVX512DQ-NEXT: vpaddb %ymm6, %ymm6, %ymm7 +; AVX512DQ-NEXT: vpblendvb %ymm7, %ymm3, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsllw $4, %ymm1, %ymm3 +; AVX512DQ-NEXT: vpand %ymm4, %ymm3, %ymm3 +; AVX512DQ-NEXT: vpblendvb %ymm2, %ymm3, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsllw $2, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpand %ymm5, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm6, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpaddb %ymm1, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm7, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq + + %splat = shufflevector <64 x i8> %b, <64 x i8> undef, <64 x i32> zeroinitializer + %shift = shl <64 x i8> %a, %splat + ret <64 x i8> %shift +} + +; +; Constant Shifts +; + +define <8 x i64> @constant_shift_v8i64(<8 x i64> %a) nounwind { +; ALL-LABEL: constant_shift_v8i64: +; ALL: ## BB#0: +; ALL-NEXT: vpsllvq {{.*}}(%rip), %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = shl <8 x i64> %a, + ret <8 x i64> %shift +} + +define <16 x i32> @constant_shift_v16i32(<16 x i32> %a) nounwind { +; ALL-LABEL: constant_shift_v16i32: +; ALL: ## BB#0: +; ALL-NEXT: vpsllvd {{.*}}(%rip), %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = shl <16 x i32> %a, + ret <16 x i32> %shift +} + +define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) nounwind { +; AVX512DQ-LABEL: constant_shift_v32i16: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm2 = [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768] +; AVX512DQ-NEXT: vpmullw %ymm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpmullw %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq +; +; AVX512BW-LABEL: constant_shift_v32i16: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsllvw {{.*}}(%rip), %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %shift = shl <32 x i16> %a, + ret <32 x i16> %shift +} + +define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) nounwind { +; AVX512DQ-LABEL: constant_shift_v64i8: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpsllw $4, %ymm0, %ymm2 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240] +; AVX512DQ-NEXT: vpand %ymm3, %ymm2, %ymm2 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0,0,1,2,3,4,5,6,7,7,6,5,4,3,2,1,0] +; AVX512DQ-NEXT: vpsllw $5, %ymm4, %ymm4 +; AVX512DQ-NEXT: vpblendvb %ymm4, %ymm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsllw $2, %ymm0, %ymm2 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm5 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252] +; AVX512DQ-NEXT: vpand %ymm5, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpaddb %ymm4, %ymm4, %ymm6 +; AVX512DQ-NEXT: vpblendvb %ymm6, %ymm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpaddb %ymm0, %ymm0, %ymm2 +; AVX512DQ-NEXT: vpaddb %ymm6, %ymm6, %ymm7 +; AVX512DQ-NEXT: vpblendvb %ymm7, %ymm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsllw $4, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpand %ymm3, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm4, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpsllw $2, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpand %ymm5, %ymm2, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm6, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpaddb %ymm1, %ymm1, %ymm2 +; AVX512DQ-NEXT: vpblendvb %ymm7, %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq + %shift = shl <64 x i8> %a, + ret <64 x i8> %shift +} + +; +; Uniform Constant Shifts +; + +define <8 x i64> @splatconstant_shift_v8i64(<8 x i64> %a) nounwind { +; ALL-LABEL: splatconstant_shift_v8i64: +; ALL: ## BB#0: +; ALL-NEXT: vpsllq $7, %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = shl <8 x i64> %a, + ret <8 x i64> %shift +} + +define <16 x i32> @splatconstant_shift_v16i32(<16 x i32> %a) nounwind { +; ALL-LABEL: splatconstant_shift_v16i32: +; ALL: ## BB#0: +; ALL-NEXT: vpslld $5, %zmm0, %zmm0 +; ALL-NEXT: retq + %shift = shl <16 x i32> %a, + ret <16 x i32> %shift +} + +define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) nounwind { +; AVX512DQ-LABEL: splatconstant_shift_v32i16: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpsllw $3, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsllw $3, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_shift_v32i16: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsllw $3, %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %shift = shl <32 x i16> %a, + ret <32 x i16> %shift +} + +define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) nounwind { +; AVX512DQ-LABEL: splatconstant_shift_v64i8: +; AVX512DQ: ## BB#0: +; AVX512DQ-NEXT: vpsllw $3, %ymm0, %ymm0 +; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm2 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248] +; AVX512DQ-NEXT: vpand %ymm2, %ymm0, %ymm0 +; AVX512DQ-NEXT: vpsllw $3, %ymm1, %ymm1 +; AVX512DQ-NEXT: vpand %ymm2, %ymm1, %ymm1 +; AVX512DQ-NEXT: retq +; +; AVX512BW-LABEL: splatconstant_shift_v64i8: +; AVX512BW: ## BB#0: +; AVX512BW-NEXT: vpsllw $3, %zmm0, %zmm0 +; AVX512BW-NEXT: vpandq {{.*}}(%rip), %zmm0, %zmm0 +; AVX512BW-NEXT: retq + %shift = shl <64 x i8> %a, + ret <64 x i8> %shift +} diff --git a/test/CodeGen/X86/vector-shuffle-128-v16.ll b/test/CodeGen/X86/vector-shuffle-128-v16.ll index 124d6e8c8ba2..13a9543ddd90 100644 --- a/test/CodeGen/X86/vector-shuffle-128-v16.ll +++ b/test/CodeGen/X86/vector-shuffle-128-v16.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 @@ -469,6 +470,20 @@ define <16 x i8> @shuffle_v16i8_00_01_02_19_04_05_06_23_08_09_10_27_12_13_14_31( ret <16 x i8> %shuffle } +define <16 x i8> @shuffle_v16i8_00_01_02_zz_04_05_06_zz_08_09_10_zz_12_13_14_zz(<16 x i8> %a) { +; SSE-LABEL: shuffle_v16i8_00_01_02_zz_04_05_06_zz_08_09_10_zz_12_13_14_zz: +; SSE: # BB#0: +; SSE-NEXT: andps {{.*}}(%rip), %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: shuffle_v16i8_00_01_02_zz_04_05_06_zz_08_09_10_zz_12_13_14_zz: +; AVX: # BB#0: +; AVX-NEXT: vandps {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: retq + %shuffle = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <16 x i32> + ret <16 x i8> %shuffle +} + define <16 x i8> @shuffle_v16i8_00_01_02_03_20_05_06_23_08_09_10_11_28_13_14_31(<16 x i8> %a, <16 x i8> %b) { ; SSE2-LABEL: shuffle_v16i8_00_01_02_03_20_05_06_23_08_09_10_11_28_13_14_31: ; SSE2: # BB#0: @@ -1356,3 +1371,264 @@ define <16 x i8> @shuffle_v16i8_bitcast_unpack(<16 x i8> %a, <16 x i8> %b) { %bitcast8 = bitcast <8 x i16> %shuffle16 to <16 x i8> ret <16 x i8> %bitcast8 } + +define <16 x i8> @insert_dup_mem_v16i8_i32(i32* %ptr) { +; SSE2-LABEL: insert_dup_mem_v16i8_i32: +; SSE2: # BB#0: +; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] +; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: insert_dup_mem_v16i8_i32: +; SSSE3: # BB#0: +; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: pshufb %xmm1, %xmm0 +; SSSE3-NEXT: retq +; +; SSE41-LABEL: insert_dup_mem_v16i8_i32: +; SSE41: # BB#0: +; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE41-NEXT: pxor %xmm1, %xmm1 +; SSE41-NEXT: pshufb %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; AVX1-LABEL: insert_dup_mem_v16i8_i32: +; AVX1: # BB#0: +; AVX1-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_mem_v16i8_i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastb (%rdi), %xmm0 +; AVX2-NEXT: retq + %tmp = load i32, i32* %ptr, align 4 + %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0 + %tmp2 = bitcast <4 x i32> %tmp1 to <16 x i8> + %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> undef, <16 x i32> zeroinitializer + ret <16 x i8> %tmp3 +} + +define <16 x i8> @insert_dup_mem_v16i8_sext_i8(i8* %ptr) { +; SSE2-LABEL: insert_dup_mem_v16i8_sext_i8: +; SSE2: # BB#0: +; SSE2-NEXT: movsbl (%rdi), %eax +; SSE2-NEXT: movd %eax, %xmm0 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] +; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: insert_dup_mem_v16i8_sext_i8: +; SSSE3: # BB#0: +; SSSE3-NEXT: movsbl (%rdi), %eax +; SSSE3-NEXT: movd %eax, %xmm0 +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: pshufb %xmm1, %xmm0 +; SSSE3-NEXT: retq +; +; SSE41-LABEL: insert_dup_mem_v16i8_sext_i8: +; SSE41: # BB#0: +; SSE41-NEXT: movsbl (%rdi), %eax +; SSE41-NEXT: movd %eax, %xmm0 +; SSE41-NEXT: pxor %xmm1, %xmm1 +; SSE41-NEXT: pshufb %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; AVX1-LABEL: insert_dup_mem_v16i8_sext_i8: +; AVX1: # BB#0: +; AVX1-NEXT: movsbl (%rdi), %eax +; AVX1-NEXT: vmovd %eax, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_mem_v16i8_sext_i8: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastb (%rdi), %xmm0 +; AVX2-NEXT: retq + %tmp = load i8, i8* %ptr, align 1 + %tmp1 = sext i8 %tmp to i32 + %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0 + %tmp3 = bitcast <4 x i32> %tmp2 to <16 x i8> + %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> undef, <16 x i32> zeroinitializer + ret <16 x i8> %tmp4 +} + +define <16 x i8> @insert_dup_elt1_mem_v16i8_i32(i32* %ptr) { +; SSE2-LABEL: insert_dup_elt1_mem_v16i8_i32: +; SSE2: # BB#0: +; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] +; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] +; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: insert_dup_elt1_mem_v16i8_i32: +; SSSE3: # BB#0: +; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: insert_dup_elt1_mem_v16i8_i32: +; SSE41: # BB#0: +; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE41-NEXT: pshufb {{.*#+}} xmm0 = xmm0[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; SSE41-NEXT: retq +; +; AVX1-LABEL: insert_dup_elt1_mem_v16i8_i32: +; AVX1: # BB#0: +; AVX1-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_elt1_mem_v16i8_i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastb 1(%rdi), %xmm0 +; AVX2-NEXT: retq + %tmp = load i32, i32* %ptr, align 4 + %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0 + %tmp2 = bitcast <4 x i32> %tmp1 to <16 x i8> + %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> undef, <16 x i32> + ret <16 x i8> %tmp3 +} + +define <16 x i8> @insert_dup_elt2_mem_v16i8_i32(i32* %ptr) { +; SSE2-LABEL: insert_dup_elt2_mem_v16i8_i32: +; SSE2: # BB#0: +; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] +; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7] +; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,6,6,6] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: insert_dup_elt2_mem_v16i8_i32: +; SSSE3: # BB#0: +; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: insert_dup_elt2_mem_v16i8_i32: +; SSE41: # BB#0: +; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE41-NEXT: pshufb {{.*#+}} xmm0 = xmm0[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2] +; SSE41-NEXT: retq +; +; AVX1-LABEL: insert_dup_elt2_mem_v16i8_i32: +; AVX1: # BB#0: +; AVX1-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2] +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_elt2_mem_v16i8_i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastb 2(%rdi), %xmm0 +; AVX2-NEXT: retq + %tmp = load i32, i32* %ptr, align 4 + %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0 + %tmp2 = bitcast <4 x i32> %tmp1 to <16 x i8> + %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> undef, <16 x i32> + ret <16 x i8> %tmp3 +} + +define <16 x i8> @insert_dup_elt1_mem_v16i8_sext_i8(i8* %ptr) { +; SSE2-LABEL: insert_dup_elt1_mem_v16i8_sext_i8: +; SSE2: # BB#0: +; SSE2-NEXT: movsbl (%rdi), %eax +; SSE2-NEXT: movd %eax, %xmm0 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] +; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] +; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: insert_dup_elt1_mem_v16i8_sext_i8: +; SSSE3: # BB#0: +; SSSE3-NEXT: movsbl (%rdi), %eax +; SSSE3-NEXT: movd %eax, %xmm0 +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: insert_dup_elt1_mem_v16i8_sext_i8: +; SSE41: # BB#0: +; SSE41-NEXT: movsbl (%rdi), %eax +; SSE41-NEXT: movd %eax, %xmm0 +; SSE41-NEXT: pshufb {{.*#+}} xmm0 = xmm0[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; SSE41-NEXT: retq +; +; AVX1-LABEL: insert_dup_elt1_mem_v16i8_sext_i8: +; AVX1: # BB#0: +; AVX1-NEXT: movsbl (%rdi), %eax +; AVX1-NEXT: vmovd %eax, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_elt1_mem_v16i8_sext_i8: +; AVX2: # BB#0: +; AVX2-NEXT: movsbl (%rdi), %eax +; AVX2-NEXT: shrl $8, %eax +; AVX2-NEXT: vmovd %eax, %xmm0 +; AVX2-NEXT: vpbroadcastb %xmm0, %xmm0 +; AVX2-NEXT: retq + %tmp = load i8, i8* %ptr, align 1 + %tmp1 = sext i8 %tmp to i32 + %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0 + %tmp3 = bitcast <4 x i32> %tmp2 to <16 x i8> + %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> undef, <16 x i32> + ret <16 x i8> %tmp4 +} + +define <16 x i8> @insert_dup_elt2_mem_v16i8_sext_i8(i8* %ptr) { +; SSE2-LABEL: insert_dup_elt2_mem_v16i8_sext_i8: +; SSE2: # BB#0: +; SSE2-NEXT: movsbl (%rdi), %eax +; SSE2-NEXT: movd %eax, %xmm0 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] +; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7] +; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,6,6,6] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: insert_dup_elt2_mem_v16i8_sext_i8: +; SSSE3: # BB#0: +; SSSE3-NEXT: movsbl (%rdi), %eax +; SSSE3-NEXT: movd %eax, %xmm0 +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: insert_dup_elt2_mem_v16i8_sext_i8: +; SSE41: # BB#0: +; SSE41-NEXT: movsbl (%rdi), %eax +; SSE41-NEXT: movd %eax, %xmm0 +; SSE41-NEXT: pshufb {{.*#+}} xmm0 = xmm0[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2] +; SSE41-NEXT: retq +; +; AVX1-LABEL: insert_dup_elt2_mem_v16i8_sext_i8: +; AVX1: # BB#0: +; AVX1-NEXT: movsbl (%rdi), %eax +; AVX1-NEXT: vmovd %eax, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2] +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_elt2_mem_v16i8_sext_i8: +; AVX2: # BB#0: +; AVX2-NEXT: movsbl (%rdi), %eax +; AVX2-NEXT: shrl $16, %eax +; AVX2-NEXT: vmovd %eax, %xmm0 +; AVX2-NEXT: vpbroadcastb %xmm0, %xmm0 +; AVX2-NEXT: retq + %tmp = load i8, i8* %ptr, align 1 + %tmp1 = sext i8 %tmp to i32 + %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0 + %tmp3 = bitcast <4 x i32> %tmp2 to <16 x i8> + %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> undef, <16 x i32> + ret <16 x i8> %tmp4 +} diff --git a/test/CodeGen/X86/vector-shuffle-128-v2.ll b/test/CodeGen/X86/vector-shuffle-128-v2.ll index ee68df581bfd..1d32f9e38523 100644 --- a/test/CodeGen/X86/vector-shuffle-128-v2.ll +++ b/test/CodeGen/X86/vector-shuffle-128-v2.ll @@ -1,9 +1,11 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE3 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl -mattr=+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512VL target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-unknown" @@ -23,6 +25,11 @@ define <2 x i64> @shuffle_v2i64_00(<2 x i64> %a, <2 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpbroadcastq %xmm0, %xmm0 ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v2i64_00: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpbroadcastq %xmm0, %xmm0 +; AVX512VL-NEXT: retq %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> ret <2 x i64> %shuffle } @@ -67,6 +74,11 @@ define <2 x i64> @shuffle_v2i64_22(<2 x i64> %a, <2 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpbroadcastq %xmm1, %xmm0 ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v2i64_22: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpbroadcastq %xmm1, %xmm0 +; AVX512VL-NEXT: retq %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> ret <2 x i64> %shuffle } @@ -135,6 +147,7 @@ define <2 x double> @shuffle_v2f64_10(<2 x double> %a, <2 x double> %b) { ; AVX: # BB#0: ; AVX-NEXT: vpermilpd {{.*#+}} xmm0 = xmm0[1,0] ; AVX-NEXT: retq + %shuffle = shufflevector <2 x double> %a, <2 x double> %b, <2 x i32> ret <2 x double> %shuffle } @@ -191,6 +204,7 @@ define <2 x double> @shuffle_v2f64_32(<2 x double> %a, <2 x double> %b) { ; AVX: # BB#0: ; AVX-NEXT: vpermilpd {{.*#+}} xmm0 = xmm1[1,0] ; AVX-NEXT: retq + %shuffle = shufflevector <2 x double> %a, <2 x double> %b, <2 x i32> ret <2 x double> %shuffle } @@ -329,6 +343,11 @@ define <2 x i64> @shuffle_v2i64_03(<2 x i64> %a, <2 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v2i64_03: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3] +; AVX512VL-NEXT: retq %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> ret <2 x i64> %shuffle } @@ -366,6 +385,11 @@ define <2 x i64> @shuffle_v2i64_03_copy(<2 x i64> %nonce, <2 x i64> %a, <2 x i64 ; AVX2: # BB#0: ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm2[2,3] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v2i64_03_copy: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm2[2,3] +; AVX512VL-NEXT: retq %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> ret <2 x i64> %shuffle } @@ -516,6 +540,11 @@ define <2 x i64> @shuffle_v2i64_21(<2 x i64> %a, <2 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v2i64_21: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3] +; AVX512VL-NEXT: retq %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> ret <2 x i64> %shuffle } @@ -553,6 +582,11 @@ define <2 x i64> @shuffle_v2i64_21_copy(<2 x i64> %nonce, <2 x i64> %a, <2 x i64 ; AVX2: # BB#0: ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm1[2,3] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v2i64_21_copy: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm1[2,3] +; AVX512VL-NEXT: retq %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> ret <2 x i64> %shuffle } @@ -725,6 +759,12 @@ define <2 x i64> @shuffle_v2i64_z1(<2 x i64> %a) { ; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v2i64_z1: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX512VL-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3] +; AVX512VL-NEXT: retq %shuffle = shufflevector <2 x i64> %a, <2 x i64> zeroinitializer, <2 x i32> ret <2 x i64> %shuffle } @@ -750,11 +790,23 @@ define <2 x double> @shuffle_v2f64_1z(<2 x double> %a) { ; SSE-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] ; SSE-NEXT: retq ; -; AVX-LABEL: shuffle_v2f64_1z: -; AVX: # BB#0: -; AVX-NEXT: vxorpd %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] -; AVX-NEXT: retq +; AVX1-LABEL: shuffle_v2f64_1z: +; AVX1: # BB#0: +; AVX1-NEXT: vxorpd %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v2f64_1z: +; AVX2: # BB#0: +; AVX2-NEXT: vxorpd %xmm1, %xmm1, %xmm1 +; AVX2-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v2f64_1z: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vxorps %xmm1, %xmm1, %xmm1 +; AVX512VL-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] +; AVX512VL-NEXT: retq %shuffle = shufflevector <2 x double> %a, <2 x double> zeroinitializer, <2 x i32> ret <2 x double> %shuffle } @@ -767,11 +819,23 @@ define <2 x double> @shuffle_v2f64_z0(<2 x double> %a) { ; SSE-NEXT: movapd %xmm1, %xmm0 ; SSE-NEXT: retq ; -; AVX-LABEL: shuffle_v2f64_z0: -; AVX: # BB#0: -; AVX-NEXT: vxorpd %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0] -; AVX-NEXT: retq +; AVX1-LABEL: shuffle_v2f64_z0: +; AVX1: # BB#0: +; AVX1-NEXT: vxorpd %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v2f64_z0: +; AVX2: # BB#0: +; AVX2-NEXT: vxorpd %xmm1, %xmm1, %xmm1 +; AVX2-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v2f64_z0: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vxorps %xmm1, %xmm1, %xmm1 +; AVX512VL-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512VL-NEXT: retq %shuffle = shufflevector <2 x double> %a, <2 x double> zeroinitializer, <2 x i32> ret <2 x double> %shuffle } @@ -817,11 +881,23 @@ define <2 x double> @shuffle_v2f64_bitcast_1z(<2 x double> %a) { ; SSE-NEXT: shufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] ; SSE-NEXT: retq ; -; AVX-LABEL: shuffle_v2f64_bitcast_1z: -; AVX: # BB#0: -; AVX-NEXT: vxorpd %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vshufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] -; AVX-NEXT: retq +; AVX1-LABEL: shuffle_v2f64_bitcast_1z: +; AVX1: # BB#0: +; AVX1-NEXT: vxorpd %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vshufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v2f64_bitcast_1z: +; AVX2: # BB#0: +; AVX2-NEXT: vxorpd %xmm1, %xmm1, %xmm1 +; AVX2-NEXT: vshufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v2f64_bitcast_1z: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX512VL-NEXT: vshufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] +; AVX512VL-NEXT: retq %shuffle64 = shufflevector <2 x double> %a, <2 x double> zeroinitializer, <2 x i32> %bitcast32 = bitcast <2 x double> %shuffle64 to <4 x float> %shuffle32 = shufflevector <4 x float> %bitcast32, <4 x float> undef, <4 x i32> @@ -829,6 +905,66 @@ define <2 x double> @shuffle_v2f64_bitcast_1z(<2 x double> %a) { ret <2 x double> %bitcast64 } +define <2 x i64> @shuffle_v2i64_bitcast_z123(<2 x i64> %x) { +; SSE2-LABEL: shuffle_v2i64_bitcast_z123: +; SSE2: # BB#0: +; SSE2-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero +; SSE2-NEXT: movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] +; SSE2-NEXT: andps {{.*}}(%rip), %xmm0 +; SSE2-NEXT: retq +; +; SSE3-LABEL: shuffle_v2i64_bitcast_z123: +; SSE3: # BB#0: +; SSE3-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero +; SSE3-NEXT: movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] +; SSE3-NEXT: andps {{.*}}(%rip), %xmm0 +; SSE3-NEXT: retq +; +; SSSE3-LABEL: shuffle_v2i64_bitcast_z123: +; SSSE3: # BB#0: +; SSSE3-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero +; SSSE3-NEXT: movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] +; SSSE3-NEXT: andps {{.*}}(%rip), %xmm0 +; SSSE3-NEXT: retq +; +; SSE41-LABEL: shuffle_v2i64_bitcast_z123: +; SSE41: # BB#0: +; SSE41-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero +; SSE41-NEXT: blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] +; SSE41-NEXT: xorps %xmm1, %xmm1 +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3,4,5,6,7] +; SSE41-NEXT: retq +; +; AVX1-LABEL: shuffle_v2i64_bitcast_z123: +; AVX1: # BB#0: +; AVX1-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero +; AVX1-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] +; AVX1-NEXT: vxorps %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3,4,5,6,7] +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v2i64_bitcast_z123: +; AVX2: # BB#0: +; AVX2-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero +; AVX2-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] +; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1 +; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v2i64_bitcast_z123: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vmovss {{.*}}(%rip), %xmm1 +; AVX512VL-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] +; AVX512VL-NEXT: vxorps %xmm1, %xmm1, %xmm1 +; AVX512VL-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] +; AVX512VL-NEXT: retq + %bitcast32 = bitcast <2 x i64> %x to <4 x float> + %shuffle32 = shufflevector <4 x float> %bitcast32, <4 x float> , <4 x i32> + %bitcast64 = bitcast <4 x float> %shuffle32 to <2 x i64> + %and = and <2 x i64> %bitcast64, + ret <2 x i64> %and +} + define <2 x i64> @insert_reg_and_zero_v2i64(i64 %a) { ; SSE-LABEL: insert_reg_and_zero_v2i64: ; SSE: # BB#0: @@ -850,10 +986,20 @@ define <2 x i64> @insert_mem_and_zero_v2i64(i64* %ptr) { ; SSE-NEXT: movq {{.*#+}} xmm0 = mem[0],zero ; SSE-NEXT: retq ; -; AVX-LABEL: insert_mem_and_zero_v2i64: -; AVX: # BB#0: -; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero -; AVX-NEXT: retq +; AVX1-LABEL: insert_mem_and_zero_v2i64: +; AVX1: # BB#0: +; AVX1-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_mem_and_zero_v2i64: +; AVX2: # BB#0: +; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: insert_mem_and_zero_v2i64: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vmovq (%rdi), %xmm0 +; AVX512VL-NEXT: retq %a = load i64, i64* %ptr %v = insertelement <2 x i64> undef, i64 %a, i32 0 %shuffle = shufflevector <2 x i64> %v, <2 x i64> zeroinitializer, <2 x i32> @@ -881,10 +1027,20 @@ define <2 x double> @insert_mem_and_zero_v2f64(double* %ptr) { ; SSE-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero ; SSE-NEXT: retq ; -; AVX-LABEL: insert_mem_and_zero_v2f64: -; AVX: # BB#0: -; AVX-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero -; AVX-NEXT: retq +; AVX1-LABEL: insert_mem_and_zero_v2f64: +; AVX1: # BB#0: +; AVX1-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_mem_and_zero_v2f64: +; AVX2: # BB#0: +; AVX2-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: insert_mem_and_zero_v2f64: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vmovsd (%rdi), %xmm0 +; AVX512VL-NEXT: retq %a = load double, double* %ptr %v = insertelement <2 x double> undef, double %a, i32 0 %shuffle = shufflevector <2 x double> %v, <2 x double> zeroinitializer, <2 x i32> @@ -927,6 +1083,12 @@ define <2 x i64> @insert_reg_lo_v2i64(i64 %a, <2 x i64> %b) { ; AVX2-NEXT: vmovq %rdi, %xmm1 ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: insert_reg_lo_v2i64: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vmovq %rdi, %xmm1 +; AVX512VL-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3] +; AVX512VL-NEXT: retq %v = insertelement <2 x i64> undef, i64 %a, i32 0 %shuffle = shufflevector <2 x i64> %v, <2 x i64> %b, <2 x i32> ret <2 x i64> %shuffle @@ -965,6 +1127,12 @@ define <2 x i64> @insert_mem_lo_v2i64(i64* %ptr, <2 x i64> %b) { ; AVX2-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: insert_mem_lo_v2i64: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vmovq (%rdi), %xmm1 +; AVX512VL-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3] +; AVX512VL-NEXT: retq %a = load i64, i64* %ptr %v = insertelement <2 x i64> undef, i64 %a, i32 0 %shuffle = shufflevector <2 x i64> %v, <2 x i64> %b, <2 x i32> @@ -995,11 +1163,23 @@ define <2 x i64> @insert_mem_hi_v2i64(i64* %ptr, <2 x i64> %b) { ; SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] ; SSE-NEXT: retq ; -; AVX-LABEL: insert_mem_hi_v2i64: -; AVX: # BB#0: -; AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero -; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] -; AVX-NEXT: retq +; AVX1-LABEL: insert_mem_hi_v2i64: +; AVX1: # BB#0: +; AVX1-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_mem_hi_v2i64: +; AVX2: # BB#0: +; AVX2-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero +; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: insert_mem_hi_v2i64: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vmovq (%rdi), %xmm1 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX512VL-NEXT: retq %a = load i64, i64* %ptr %v = insertelement <2 x i64> undef, i64 %a, i32 0 %shuffle = shufflevector <2 x i64> %v, <2 x i64> %b, <2 x i32> @@ -1013,10 +1193,20 @@ define <2 x double> @insert_reg_lo_v2f64(double %a, <2 x double> %b) { ; SSE-NEXT: movapd %xmm1, %xmm0 ; SSE-NEXT: retq ; -; AVX-LABEL: insert_reg_lo_v2f64: -; AVX: # BB#0: -; AVX-NEXT: vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1] -; AVX-NEXT: retq +; AVX1-LABEL: insert_reg_lo_v2f64: +; AVX1: # BB#0: +; AVX1-NEXT: vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1] +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_reg_lo_v2f64: +; AVX2: # BB#0: +; AVX2-NEXT: vmovsd {{.*#+}} xmm0 = xmm0[0],xmm1[1] +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: insert_reg_lo_v2f64: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vmovsd %xmm0, %xmm1, %xmm0 +; AVX512VL-NEXT: retq %v = insertelement <2 x double> undef, double %a, i32 0 %shuffle = shufflevector <2 x double> %v, <2 x double> %b, <2 x i32> ret <2 x double> %shuffle @@ -1071,8 +1261,6 @@ define <2 x double> @insert_mem_hi_v2f64(double* %ptr, <2 x double> %b) { } define <2 x double> @insert_dup_reg_v2f64(double %a) { -; FIXME: We should match movddup for SSE3 and higher here. -; ; SSE2-LABEL: insert_dup_reg_v2f64: ; SSE2: # BB#0: ; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0,0] @@ -1101,6 +1289,7 @@ define <2 x double> @insert_dup_reg_v2f64(double %a) { %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> ret <2 x double> %shuffle } + define <2 x double> @insert_dup_mem_v2f64(double* %ptr) { ; SSE2-LABEL: insert_dup_mem_v2f64: ; SSE2: # BB#0: @@ -1133,6 +1322,66 @@ define <2 x double> @insert_dup_mem_v2f64(double* %ptr) { ret <2 x double> %shuffle } +define <2 x double> @insert_dup_mem128_v2f64(<2 x double>* %ptr) nounwind { +; SSE2-LABEL: insert_dup_mem128_v2f64: +; SSE2: # BB#0: +; SSE2-NEXT: movaps (%rdi), %xmm0 +; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0,0] +; SSE2-NEXT: retq +; +; SSE3-LABEL: insert_dup_mem128_v2f64: +; SSE3: # BB#0: +; SSE3-NEXT: movddup {{.*#+}} xmm0 = mem[0,0] +; SSE3-NEXT: retq +; +; SSSE3-LABEL: insert_dup_mem128_v2f64: +; SSSE3: # BB#0: +; SSSE3-NEXT: movddup {{.*#+}} xmm0 = mem[0,0] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: insert_dup_mem128_v2f64: +; SSE41: # BB#0: +; SSE41-NEXT: movddup {{.*#+}} xmm0 = mem[0,0] +; SSE41-NEXT: retq +; +; AVX-LABEL: insert_dup_mem128_v2f64: +; AVX: # BB#0: +; AVX-NEXT: vmovddup {{.*#+}} xmm0 = mem[0,0] +; AVX-NEXT: retq + %v = load <2 x double>, <2 x double>* %ptr + %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> + ret <2 x double> %shuffle +} + + +define <2 x i64> @insert_dup_mem_v2i64(i64* %ptr) { +; SSE-LABEL: insert_dup_mem_v2i64: +; SSE: # BB#0: +; SSE-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; SSE-NEXT: retq +; +; AVX1-LABEL: insert_dup_mem_v2i64: +; AVX1: # BB#0: +; AVX1-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_mem_v2i64: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastq (%rdi), %xmm0 +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: insert_dup_mem_v2i64: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpbroadcastq (%rdi), %xmm0 +; AVX512VL-NEXT: retq + %tmp = load i64, i64* %ptr, align 1 + %tmp1 = insertelement <2 x i64> undef, i64 %tmp, i32 0 + %tmp2 = shufflevector <2 x i64> %tmp1, <2 x i64> undef, <2 x i32> zeroinitializer + ret <2 x i64> %tmp2 +} + define <2 x double> @shuffle_mem_v2f64_10(<2 x double>* %ptr) { ; SSE-LABEL: shuffle_mem_v2f64_10: ; SSE: # BB#0: @@ -1144,6 +1393,7 @@ define <2 x double> @shuffle_mem_v2f64_10(<2 x double>* %ptr) { ; AVX: # BB#0: ; AVX-NEXT: vpermilpd {{.*#+}} xmm0 = mem[1,0] ; AVX-NEXT: retq + %a = load <2 x double>, <2 x double>* %ptr %shuffle = shufflevector <2 x double> %a, <2 x double> undef, <2 x i32> ret <2 x double> %shuffle diff --git a/test/CodeGen/X86/vector-shuffle-128-v4.ll b/test/CodeGen/X86/vector-shuffle-128-v4.ll index 8612a5afa3d2..35c3b708fd08 100644 --- a/test/CodeGen/X86/vector-shuffle-128-v4.ll +++ b/test/CodeGen/X86/vector-shuffle-128-v4.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE3 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3 @@ -952,6 +953,43 @@ define <4 x float> @shuffle_v4f32_0zz3(<4 x float> %a) { ret <4 x float> %shuffle } +define <4 x float> @shuffle_v4f32_0z2z(<4 x float> %v) { +; SSE2-LABEL: shuffle_v4f32_0z2z: +; SSE2: # BB#0: +; SSE2-NEXT: xorps %xmm1, %xmm1 +; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,0] +; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2,1,3] +; SSE2-NEXT: retq +; +; SSE3-LABEL: shuffle_v4f32_0z2z: +; SSE3: # BB#0: +; SSE3-NEXT: xorps %xmm1, %xmm1 +; SSE3-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,0] +; SSE3-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2,1,3] +; SSE3-NEXT: retq +; +; SSSE3-LABEL: shuffle_v4f32_0z2z: +; SSSE3: # BB#0: +; SSSE3-NEXT: xorps %xmm1, %xmm1 +; SSSE3-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,0] +; SSSE3-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2,1,3] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: shuffle_v4f32_0z2z: +; SSE41: # BB#0: +; SSE41-NEXT: xorps %xmm1, %xmm1 +; SSE41-NEXT: blendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] +; SSE41-NEXT: retq +; +; AVX-LABEL: shuffle_v4f32_0z2z: +; AVX: # BB#0: +; AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] +; AVX-NEXT: retq + %shuffle = shufflevector <4 x float> %v, <4 x float> , <4 x i32> + ret <4 x float> %shuffle +} + define <4 x float> @shuffle_v4f32_u051(<4 x float> %a, <4 x float> %b) { ; SSE-LABEL: shuffle_v4f32_u051: ; SSE: # BB#0: @@ -1591,6 +1629,43 @@ define <4 x i32> @shuffle_v4i32_bitcast_0415(<4 x i32> %a, <4 x i32> %b) { ret <4 x i32> %bitcast32 } +define <4 x float> @shuffle_v4f32_bitcast_4401(<4 x float> %a, <4 x i32> %b) { +; SSE-LABEL: shuffle_v4f32_bitcast_4401: +; SSE: # BB#0: +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1] +; SSE-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; SSE-NEXT: movapd %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: shuffle_v4f32_bitcast_4401: +; AVX: # BB#0: +; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,1,1] +; AVX-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX-NEXT: retq + %1 = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> + %2 = bitcast <4 x i32> %1 to <2 x double> + %3 = bitcast <4 x float> %a to <2 x double> + %4 = shufflevector <2 x double> %2, <2 x double> %3, <2 x i32> + %5 = bitcast <2 x double> %4 to <4 x float> + ret <4 x float> %5 +} + +define <4 x float> @shuffle_v4f32_bitcast_0045(<4 x float> %a, <4 x i32> %b) { +; SSE-LABEL: shuffle_v4f32_bitcast_0045: +; SSE: # BB#0: +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,1] +; SSE-NEXT: retq +; +; AVX-LABEL: shuffle_v4f32_bitcast_0045: +; AVX: # BB#0: +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,1] +; AVX-NEXT: retq + %1 = shufflevector <4 x float> %a, <4 x float> undef, <4 x i32> + %2 = bitcast <4 x i32> %b to <4 x float> + %3 = shufflevector <4 x float> %1, <4 x float> %2, <4 x i32> + ret <4 x float> %3 +} + define <4 x i32> @insert_reg_and_zero_v4i32(i32 %a) { ; SSE-LABEL: insert_reg_and_zero_v4i32: ; SSE: # BB#0: @@ -1875,6 +1950,23 @@ define <4 x float> @shuffle_mem_v4f32_3210(<4 x float>* %ptr) { ret <4 x float> %shuffle } +define <4 x i32> @insert_dup_mem_v4i32(i32* %ptr) { +; SSE-LABEL: insert_dup_mem_v4i32: +; SSE: # BB#0: +; SSE-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0] +; SSE-NEXT: retq +; +; AVX-LABEL: insert_dup_mem_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vbroadcastss (%rdi), %xmm0 +; AVX-NEXT: retq + %tmp = load i32, i32* %ptr, align 4 + %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0 + %tmp2 = shufflevector <4 x i32> %tmp1, <4 x i32> undef, <4 x i32> zeroinitializer + ret <4 x i32> %tmp2 +} + ; ; Shuffle to logical bit shifts ; diff --git a/test/CodeGen/X86/vector-shuffle-128-v8.ll b/test/CodeGen/X86/vector-shuffle-128-v8.ll index 6a29d33d6c5e..168b3e33bfcf 100644 --- a/test/CodeGen/X86/vector-shuffle-128-v8.ll +++ b/test/CodeGen/X86/vector-shuffle-128-v8.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 @@ -2145,3 +2146,254 @@ define <8 x i16> @shuffle_v8i16_8012345u(<8 x i16> %a) { ret <8 x i16> %shuffle } + +define <8 x i16> @insert_dup_mem_v8i16_i32(i32* %ptr) { +; SSE2-LABEL: insert_dup_mem_v8i16_i32: +; SSE2: # BB#0: +; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] +; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: insert_dup_mem_v8i16_i32: +; SSSE3: # BB#0: +; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: insert_dup_mem_v8i16_i32: +; SSE41: # BB#0: +; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE41-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1] +; SSE41-NEXT: retq +; +; AVX1-LABEL: insert_dup_mem_v8i16_i32: +; AVX1: # BB#0: +; AVX1-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1] +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_mem_v8i16_i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastw (%rdi), %xmm0 +; AVX2-NEXT: retq + %tmp = load i32, i32* %ptr, align 4 + %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0 + %tmp2 = bitcast <4 x i32> %tmp1 to <8 x i16> + %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <8 x i32> zeroinitializer + ret <8 x i16> %tmp3 +} + +define <8 x i16> @insert_dup_mem_v8i16_sext_i16(i16* %ptr) { +; SSE2-LABEL: insert_dup_mem_v8i16_sext_i16: +; SSE2: # BB#0: +; SSE2-NEXT: movswl (%rdi), %eax +; SSE2-NEXT: movd %eax, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] +; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: insert_dup_mem_v8i16_sext_i16: +; SSSE3: # BB#0: +; SSSE3-NEXT: movswl (%rdi), %eax +; SSSE3-NEXT: movd %eax, %xmm0 +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: insert_dup_mem_v8i16_sext_i16: +; SSE41: # BB#0: +; SSE41-NEXT: movswl (%rdi), %eax +; SSE41-NEXT: movd %eax, %xmm0 +; SSE41-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1] +; SSE41-NEXT: retq +; +; AVX1-LABEL: insert_dup_mem_v8i16_sext_i16: +; AVX1: # BB#0: +; AVX1-NEXT: movswl (%rdi), %eax +; AVX1-NEXT: vmovd %eax, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1] +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_mem_v8i16_sext_i16: +; AVX2: # BB#0: +; AVX2-NEXT: movswl (%rdi), %eax +; AVX2-NEXT: vmovd %eax, %xmm0 +; AVX2-NEXT: vpbroadcastw %xmm0, %xmm0 +; AVX2-NEXT: retq + %tmp = load i16, i16* %ptr, align 2 + %tmp1 = sext i16 %tmp to i32 + %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0 + %tmp3 = bitcast <4 x i32> %tmp2 to <8 x i16> + %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> undef, <8 x i32> zeroinitializer + ret <8 x i16> %tmp4 +} + +define <8 x i16> @insert_dup_elt1_mem_v8i16_i32(i32* %ptr) { +; SSE2-LABEL: insert_dup_elt1_mem_v8i16_i32: +; SSE2: # BB#0: +; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] +; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] +; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: insert_dup_elt1_mem_v8i16_i32: +; SSSE3: # BB#0: +; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: insert_dup_elt1_mem_v8i16_i32: +; SSE41: # BB#0: +; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE41-NEXT: pshufb {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3] +; SSE41-NEXT: retq +; +; AVX1-LABEL: insert_dup_elt1_mem_v8i16_i32: +; AVX1: # BB#0: +; AVX1-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3] +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_elt1_mem_v8i16_i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastw 2(%rdi), %xmm0 +; AVX2-NEXT: retq + %tmp = load i32, i32* %ptr, align 4 + %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0 + %tmp2 = bitcast <4 x i32> %tmp1 to <8 x i16> + %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <8 x i32> + ret <8 x i16> %tmp3 +} + +define <8 x i16> @insert_dup_elt3_mem_v8i16_i32(i32* %ptr) { +; SSE2-LABEL: insert_dup_elt3_mem_v8i16_i32: +; SSE2: # BB#0: +; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,0,1,0] +; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7] +; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: insert_dup_elt3_mem_v8i16_i32: +; SSSE3: # BB#0: +; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: insert_dup_elt3_mem_v8i16_i32: +; SSE41: # BB#0: +; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE41-NEXT: pshufb {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3] +; SSE41-NEXT: retq +; +; AVX1-LABEL: insert_dup_elt3_mem_v8i16_i32: +; AVX1: # BB#0: +; AVX1-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3] +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_elt3_mem_v8i16_i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastw 2(%rdi), %xmm0 +; AVX2-NEXT: retq + %tmp = load i32, i32* %ptr, align 4 + %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 1 + %tmp2 = bitcast <4 x i32> %tmp1 to <8 x i16> + %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <8 x i32> + ret <8 x i16> %tmp3 +} + +define <8 x i16> @insert_dup_elt1_mem_v8i16_sext_i16(i16* %ptr) { +; SSE2-LABEL: insert_dup_elt1_mem_v8i16_sext_i16: +; SSE2: # BB#0: +; SSE2-NEXT: movswl (%rdi), %eax +; SSE2-NEXT: movd %eax, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] +; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] +; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: insert_dup_elt1_mem_v8i16_sext_i16: +; SSSE3: # BB#0: +; SSSE3-NEXT: movswl (%rdi), %eax +; SSSE3-NEXT: movd %eax, %xmm0 +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: insert_dup_elt1_mem_v8i16_sext_i16: +; SSE41: # BB#0: +; SSE41-NEXT: movswl (%rdi), %eax +; SSE41-NEXT: movd %eax, %xmm0 +; SSE41-NEXT: pshufb {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3] +; SSE41-NEXT: retq +; +; AVX1-LABEL: insert_dup_elt1_mem_v8i16_sext_i16: +; AVX1: # BB#0: +; AVX1-NEXT: movswl (%rdi), %eax +; AVX1-NEXT: vmovd %eax, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3] +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_elt1_mem_v8i16_sext_i16: +; AVX2: # BB#0: +; AVX2-NEXT: movswl (%rdi), %eax +; AVX2-NEXT: shrl $16, %eax +; AVX2-NEXT: vmovd %eax, %xmm0 +; AVX2-NEXT: vpbroadcastw %xmm0, %xmm0 +; AVX2-NEXT: retq + %tmp = load i16, i16* %ptr, align 2 + %tmp1 = sext i16 %tmp to i32 + %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0 + %tmp3 = bitcast <4 x i32> %tmp2 to <8 x i16> + %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> undef, <8 x i32> + ret <8 x i16> %tmp4 +} + +define <8 x i16> @insert_dup_elt3_mem_v8i16_sext_i16(i16* %ptr) { +; SSE2-LABEL: insert_dup_elt3_mem_v8i16_sext_i16: +; SSE2: # BB#0: +; SSE2-NEXT: movswl (%rdi), %eax +; SSE2-NEXT: movd %eax, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,0,1,0] +; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7] +; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: insert_dup_elt3_mem_v8i16_sext_i16: +; SSSE3: # BB#0: +; SSSE3-NEXT: movswl (%rdi), %eax +; SSSE3-NEXT: movd %eax, %xmm0 +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: insert_dup_elt3_mem_v8i16_sext_i16: +; SSE41: # BB#0: +; SSE41-NEXT: movswl (%rdi), %eax +; SSE41-NEXT: movd %eax, %xmm0 +; SSE41-NEXT: pshufb {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3] +; SSE41-NEXT: retq +; +; AVX1-LABEL: insert_dup_elt3_mem_v8i16_sext_i16: +; AVX1: # BB#0: +; AVX1-NEXT: movswl (%rdi), %eax +; AVX1-NEXT: vmovd %eax, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3] +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_elt3_mem_v8i16_sext_i16: +; AVX2: # BB#0: +; AVX2-NEXT: movswl (%rdi), %eax +; AVX2-NEXT: shrl $16, %eax +; AVX2-NEXT: vmovd %eax, %xmm0 +; AVX2-NEXT: vpbroadcastw %xmm0, %xmm0 +; AVX2-NEXT: retq + %tmp = load i16, i16* %ptr, align 2 + %tmp1 = sext i16 %tmp to i32 + %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 1 + %tmp3 = bitcast <4 x i32> %tmp2 to <8 x i16> + %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> undef, <8 x i32> + ret <8 x i16> %tmp4 +} diff --git a/test/CodeGen/X86/vector-shuffle-256-v16.ll b/test/CodeGen/X86/vector-shuffle-256-v16.ll index df4994da6932..7e3dc6e294f8 100644 --- a/test/CodeGen/X86/vector-shuffle-256-v16.ll +++ b/test/CodeGen/X86/vector-shuffle-256-v16.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX1 ; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX2 @@ -158,11 +159,11 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_08_00_00_00_00_00_00_00_0 ; ; AVX2-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_08_00_00_00_00_00_00_00_00: ; AVX2: # BB#0: -; AVX2-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm0[2,3,0,1] -; AVX2-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,16,17,16,17,16,17,16,17,16,17,16,17,16,17,16,17] -; AVX2-NEXT: vpbroadcastw %xmm0, %ymm0 +; AVX2-NEXT: vpbroadcastw %xmm0, %xmm1 +; AVX2-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3,0,1] +; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,16,17,16,17,16,17,16,17,16,17,16,17,16,17,16,17] ; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] -; AVX2-NEXT: vpblendvb %ymm2, %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0 ; AVX2-NEXT: retq %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> ret <16 x i16> %shuffle @@ -1439,11 +1440,10 @@ define <16 x i16> @shuffle_v16i16_02_03_zz_zz_06_07_zz_zz_10_11_zz_zz_14_15_zz_z define <16 x i16> @shuffle_v16i16_16_zz_zz_zz_17_zz_zz_zz_18_zz_zz_zz_19_zz_zz_zz(<16 x i16> %a) { ; AVX1-LABEL: shuffle_v16i16_16_zz_zz_zz_17_zz_zz_zz_18_zz_zz_zz_19_zz_zz_zz: ; AVX1: # BB#0: -; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[4,5,2,3,4,5,6,7,6,7,10,11,4,5,6,7] -; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7] +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] ; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: shuffle_v16i16_16_zz_zz_zz_17_zz_zz_zz_18_zz_zz_zz_19_zz_zz_zz: @@ -1702,21 +1702,21 @@ define <16 x i16> @shuffle_v16i16_uu_00_uu_01_uu_02_uu_11_uu_08_uu_09_uu_10_uu_1 ; AVX1-LABEL: shuffle_v16i16_uu_00_uu_01_uu_02_uu_11_uu_08_uu_09_uu_10_uu_11: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] ; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,2,4,5,6,7] ; AVX1-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,6,7] -; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: shuffle_v16i16_uu_00_uu_01_uu_02_uu_11_uu_08_uu_09_uu_10_uu_11: ; AVX2: # BB#0: ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] ; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,2,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,6,7] -; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3] -; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 ; AVX2-NEXT: retq %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> ret <16 x i16> %shuffle @@ -1726,21 +1726,21 @@ define <16 x i16> @shuffle_v16i16_uu_04_uu_05_uu_06_uu_15_uu_12_uu_13_uu_14_uu_1 ; AVX1-LABEL: shuffle_v16i16_uu_04_uu_05_uu_06_uu_15_uu_12_uu_13_uu_14_uu_15: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] ; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,2,4,5,6,7] ; AVX1-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,6,7] -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: shuffle_v16i16_uu_04_uu_05_uu_06_uu_15_uu_12_uu_13_uu_14_uu_15: ; AVX2: # BB#0: ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] ; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] ; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,2,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,6,7] -; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] -; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 ; AVX2-NEXT: retq %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> ret <16 x i16> %shuffle @@ -2444,13 +2444,13 @@ define <16 x i16> @shuffle_v16i16_00_16_01_17_02_18_03_27_08_24_09_25_10_26_11_2 ; ; AVX2-LABEL: shuffle_v16i16_00_16_01_17_02_18_03_27_08_24_09_25_10_26_11_27: ; AVX2: # BB#0: -; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11] ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] ; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,2,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,6,7] -; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3] -; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1 +; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11] ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15] ; AVX2-NEXT: retq %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> @@ -2498,13 +2498,13 @@ define <16 x i16> @shuffle_v16i16_04_20_05_21_06_22_07_31_12_28_13_29_14_30_15_3 ; ; AVX2-LABEL: shuffle_v16i16_04_20_05_21_06_22_07_31_12_28_13_29_14_30_15_31: ; AVX2: # BB#0: -; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15] ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 +; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] ; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] ; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,2,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,6,7] -; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] -; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1 +; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15] ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15] ; AVX2-NEXT: retq %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> @@ -2647,13 +2647,13 @@ define <16 x i16> @shuffle_v16i16_16_00_17_01_18_02_19_11_24_08_25_09_26_10_27_1 ; ; AVX2-LABEL: shuffle_v16i16_16_00_17_01_18_02_19_11_24_08_25_09_26_10_27_11: ; AVX2: # BB#0: -; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2 +; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] ; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,2,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,6,7] -; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3] -; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0 +; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11] ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7],ymm1[8],ymm0[9],ymm1[10],ymm0[11],ymm1[12],ymm0[13],ymm1[14],ymm0[15] ; AVX2-NEXT: retq %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> @@ -2674,13 +2674,13 @@ define <16 x i16> @shuffle_v16i16_20_04_21_05_22_06_23_15_28_12_29_13_30_14_31_1 ; ; AVX2-LABEL: shuffle_v16i16_20_04_21_05_22_06_23_15_28_12_29_13_30_14_31_15: ; AVX2: # BB#0: -; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15] ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2 +; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] ; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] ; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,2,4,5,6,7] ; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,6,7] -; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7] -; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0 +; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15] ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7],ymm1[8],ymm0[9],ymm1[10],ymm0[11],ymm1[12],ymm0[13],ymm1[14],ymm0[15] ; AVX2-NEXT: retq %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> @@ -3250,6 +3250,90 @@ define <16 x i16> @shuffle_v16i16_23_uu_03_uu_20_20_05_uu_31_uu_11_uu_28_28_13_u ret <16 x i16> %shuffle } +define <16 x i16> @shuffle_v16i16_u_u_u_u_u_u_u_u_0_16_1_17_2_18_3_19(<16 x i16> %a, <16 x i16> %b) { +; AVX1-LABEL: shuffle_v16i16_u_u_u_u_u_u_u_u_0_16_1_17_2_18_3_19: +; AVX1: # BB#0: +; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v16i16_u_u_u_u_u_u_u_u_0_16_1_17_2_18_3_19: +; AVX2: # BB#0: +; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 +; AVX2-NEXT: retq + %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> + ret <16 x i16> %shuffle +} + +define <16 x i16> @shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3(<16 x i16> %a, <16 x i16> %b) { +; AVX1-LABEL: shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3: +; AVX1: # BB#0: +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[6,7,6,7,6,7,6,7,6,7,6,7,6,7,6,7] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3: +; AVX2: # BB#0: +; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[6,7,6,7,6,7,6,7,6,7,6,7,6,7,6,7] +; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 +; AVX2-NEXT: retq + %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> + ret <16 x i16> %shuffle +} + +define <16 x i16> @shuffle_v16i16_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8(<16 x i16> %a, <16 x i16> %b) { +; AVX1-LABEL: shuffle_v16i16_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v16i16_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8: +; AVX2: # BB#0: +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 +; AVX2-NEXT: vpbroadcastw %xmm0, %xmm0 +; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 +; AVX2-NEXT: retq + %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> + ret <16 x i16> %shuffle +} + +define <16 x i16> @shuffle_v16i16_4_20_5_21_6_22_7_23_u_u_u_u_u_u_u_u(<16 x i16> %a, <16 x i16> %b) { +; ALL-LABEL: shuffle_v16i16_4_20_5_21_6_22_7_23_u_u_u_u_u_u_u_u: +; ALL: # BB#0: +; ALL-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; ALL-NEXT: retq + %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> + ret <16 x i16> %shuffle +} + +define <16 x i16> @shuffle_v16i16_3_3_3_3_3_3_3_3_u_u_u_u_u_u_u_u(<16 x i16> %a, <16 x i16> %b) { +; ALL-LABEL: shuffle_v16i16_3_3_3_3_3_3_3_3_u_u_u_u_u_u_u_u: +; ALL: # BB#0: +; ALL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[6,7,6,7,6,7,6,7,6,7,6,7,6,7,6,7] +; ALL-NEXT: retq + %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> + ret <16 x i16> %shuffle +} + +define <16 x i16> @shuffle_v16i16_9_9_9_9_9_9_9_9_u_u_u_u_u_u_u_u(<16 x i16> %a, <16 x i16> %b) { +; AVX1-LABEL: shuffle_v16i16_9_9_9_9_9_9_9_9_u_u_u_u_u_u_u_u: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3] +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v16i16_9_9_9_9_9_9_9_9_u_u_u_u_u_u_u_u: +; AVX2: # BB#0: +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 +; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3] +; AVX2-NEXT: retq + %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> + ret <16 x i16> %shuffle +} + define <16 x i16> @insert_v16i16_0elt_into_zero_vector(i16* %ptr) { ; ALL-LABEL: insert_v16i16_0elt_into_zero_vector: ; ALL: # BB#0: @@ -3261,3 +3345,112 @@ define <16 x i16> @insert_v16i16_0elt_into_zero_vector(i16* %ptr) { ret <16 x i16> %i0 } +define <16 x i16> @concat_v16i16_0_1_2_3_4_5_6_7_24_25_26_27_28_29_30_31(<16 x i16> %a, <16 x i16> %b) { +; AVX1-LABEL: concat_v16i16_0_1_2_3_4_5_6_7_24_25_26_27_28_29_30_31: +; AVX1: # BB#0: +; AVX1-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3] +; AVX1-NEXT: retq +; +; AVX2-LABEL: concat_v16i16_0_1_2_3_4_5_6_7_24_25_26_27_28_29_30_31: +; AVX2: # BB#0: +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; AVX2-NEXT: retq + %alo = shufflevector <16 x i16> %a, <16 x i16> undef, <8 x i32> + %bhi = shufflevector <16 x i16> %b, <16 x i16> undef, <8 x i32> + %shuf = shufflevector <8 x i16> %alo, <8 x i16> %bhi, <16 x i32> + ret <16 x i16> %shuf +} + +define <16 x i16> @concat_v16i16_8_9_10_11_12_13_14_15_24_25_26_27_28_29_30_31_bc(<16 x i16> %a, <16 x i16> %b) { +; ALL-LABEL: concat_v16i16_8_9_10_11_12_13_14_15_24_25_26_27_28_29_30_31_bc: +; ALL: # BB#0: +; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3] +; ALL-NEXT: retq + %ahi = shufflevector <16 x i16> %a, <16 x i16> undef, <8 x i32> + %bhi = shufflevector <16 x i16> %b, <16 x i16> undef, <8 x i32> + %bc0hi = bitcast <8 x i16> %ahi to <16 x i8> + %bc1hi = bitcast <8 x i16> %bhi to <16 x i8> + %shuffle8 = shufflevector <16 x i8> %bc0hi, <16 x i8> %bc1hi, <32 x i32> + %shuffle16 = bitcast <32 x i8> %shuffle8 to <16 x i16> + ret <16 x i16> %shuffle16 +} + +define <16 x i16> @insert_dup_mem_v16i16_i32(i32* %ptr) { +; AVX1-LABEL: insert_dup_mem_v16i16_i32: +; AVX1: # BB#0: +; AVX1-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_mem_v16i16_i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastw (%rdi), %ymm0 +; AVX2-NEXT: retq + %tmp = load i32, i32* %ptr, align 4 + %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0 + %tmp2 = bitcast <4 x i32> %tmp1 to <8 x i16> + %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <16 x i32> zeroinitializer + ret <16 x i16> %tmp3 +} + +define <16 x i16> @insert_dup_mem_v16i16_sext_i16(i16* %ptr) { +; AVX1-LABEL: insert_dup_mem_v16i16_sext_i16: +; AVX1: # BB#0: +; AVX1-NEXT: movswl (%rdi), %eax +; AVX1-NEXT: vmovd %eax, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_mem_v16i16_sext_i16: +; AVX2: # BB#0: +; AVX2-NEXT: movswl (%rdi), %eax +; AVX2-NEXT: vmovd %eax, %xmm0 +; AVX2-NEXT: vpbroadcastw %xmm0, %ymm0 +; AVX2-NEXT: retq + %tmp = load i16, i16* %ptr, align 2 + %tmp1 = sext i16 %tmp to i32 + %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0 + %tmp3 = bitcast <4 x i32> %tmp2 to <8 x i16> + %tmp4 = shufflevector <8 x i16> %tmp3, <8 x i16> undef, <16 x i32> zeroinitializer + ret <16 x i16> %tmp4 +} + +define <16 x i16> @insert_dup_elt1_mem_v16i16_i32(i32* %ptr) #0 { +; AVX1-LABEL: insert_dup_elt1_mem_v16i16_i32: +; AVX1: # BB#0: +; AVX1-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_elt1_mem_v16i16_i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastw 2(%rdi), %ymm0 +; AVX2-NEXT: retq + %tmp = load i32, i32* %ptr, align 4 + %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0 + %tmp2 = bitcast <4 x i32> %tmp1 to <8 x i16> + %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <16 x i32> + ret <16 x i16> %tmp3 +} + +define <16 x i16> @insert_dup_elt3_mem_v16i16_i32(i32* %ptr) #0 { +; AVX1-LABEL: insert_dup_elt3_mem_v16i16_i32: +; AVX1: # BB#0: +; AVX1-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_elt3_mem_v16i16_i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastw 2(%rdi), %ymm0 +; AVX2-NEXT: retq + %tmp = load i32, i32* %ptr, align 4 + %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 1 + %tmp2 = bitcast <4 x i32> %tmp1 to <8 x i16> + %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <16 x i32> + ret <16 x i16> %tmp3 +} diff --git a/test/CodeGen/X86/vector-shuffle-256-v32.ll b/test/CodeGen/X86/vector-shuffle-256-v32.ll index a0f43de75630..161a21cef030 100644 --- a/test/CodeGen/X86/vector-shuffle-256-v32.ll +++ b/test/CodeGen/X86/vector-shuffle-256-v32.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 ; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 @@ -324,7 +325,7 @@ define <32 x i8> @shuffle_v32i8_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_16_ ; AVX2-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm0[2,3,0,1] ; AVX2-NEXT: vpxor %ymm2, %ymm2, %ymm2 ; AVX2-NEXT: vpshufb %ymm2, %ymm1, %ymm1 -; AVX2-NEXT: vpbroadcastb %xmm0, %ymm0 +; AVX2-NEXT: vpbroadcastb %xmm0, %xmm0 ; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] ; AVX2-NEXT: vpblendvb %ymm2, %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq @@ -947,6 +948,24 @@ define <32 x i8> @shuffle_v32i8_32_01_34_03_36_05_38_07_40_09_42_11_44_13_46_15_ ret <32 x i8> %shuffle } +define <32 x i8> @shuffle_v32i8_zz_01_zz_03_zz_05_zz_07_zz_09_zz_11_zz_13_zz_15_zz_17_zz_19_zz_21_zz_23_zz_25_zz_27_zz_29_zz_31(<32 x i8> %a) { +; AVX1-LABEL: shuffle_v32i8_zz_01_zz_03_zz_05_zz_07_zz_09_zz_11_zz_13_zz_15_zz_17_zz_19_zz_21_zz_23_zz_25_zz_27_zz_29_zz_31: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vmovaps {{.*#+}} xmm2 = [0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255] +; AVX1-NEXT: vandps %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vandps %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v32i8_zz_01_zz_03_zz_05_zz_07_zz_09_zz_11_zz_13_zz_15_zz_17_zz_19_zz_21_zz_23_zz_25_zz_27_zz_29_zz_31: +; AVX2: # BB#0: +; AVX2-NEXT: vandps {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: retq + %shuffle = shufflevector <32 x i8> %a, <32 x i8> zeroinitializer, <32 x i32> + ret <32 x i8> %shuffle +} + define <32 x i8> @shuffle_v32i8_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32(<32 x i8> %a, <32 x i8> %b) { ; AVX1-LABEL: shuffle_v32i8_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32_00_32: ; AVX1: # BB#0: @@ -1737,7 +1756,8 @@ define <32 x i8> @shuffle_v32i8_32_zz_zz_zz_zz_zz_zz_zz_33_zz_zz_zz_zz_zz_zz_zz_ ; AVX1-LABEL: shuffle_v32i8_32_zz_zz_zz_zz_zz_zz_zz_33_zz_zz_zz_zz_zz_zz_zz_34_zz_zz_zz_zz_zz_zz_zz_35_zz_zz_zz_zz_zz_zz_zz: ; AVX1: # BB#0: ; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero -; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 ; AVX1-NEXT: retq ; @@ -1754,7 +1774,8 @@ define <32 x i8> @shuffle_v32i8_32_zz_zz_zz_33_zz_zz_zz_34_zz_zz_zz_35_zz_zz_zz_ ; AVX1-LABEL: shuffle_v32i8_32_zz_zz_zz_33_zz_zz_zz_34_zz_zz_zz_35_zz_zz_zz_36_zz_zz_zz_37_zz_zz_zz_38_zz_zz_zz_39_zz_zz_zz: ; AVX1: # BB#0: ; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero -; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 ; AVX1-NEXT: retq ; @@ -1956,3 +1977,186 @@ define <32 x i8> @shuffle_v32i8_15_00_01_02_03_04_05_06_07_08_09_10_11_12_13_14_ %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> ret <32 x i8> %shuffle } + +define <32 x i8> @shuffle_v32i8_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_10_10_10_10_10_10_10_10_10_10_10_10_10_10_10_10(<32 x i8> %a, <32 x i8> %b) { +; AVX1-LABEL: shuffle_v32i8_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_10_10_10_10_10_10_10_10_10_10_10_10_10_10_10_10: +; AVX1: # BB#0: +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v32i8_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_10_10_10_10_10_10_10_10_10_10_10_10_10_10_10_10: +; AVX2: # BB#0: +; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10] +; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 +; AVX2-NEXT: retq + %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> + ret <32 x i8> %shuffle +} + +define <32 x i8> @shuffle_v32i8_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16(<32 x i8> %a, <32 x i8> %b) { +; AVX1-LABEL: shuffle_v32i8_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v32i8_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16: +; AVX2: # BB#0: +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 +; AVX2-NEXT: vpbroadcastb %xmm0, %xmm0 +; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 +; AVX2-NEXT: retq + %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> + ret <32 x i8> %shuffle +} + +define <32 x i8> @shuffle_v32i8_15_15_15_15_15_15_15_15_32_32_32_32_32_32_32_32_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu(<32 x i8> %a, <32 x i8> %b) { +; AVX1-LABEL: shuffle_v32i8_15_15_15_15_15_15_15_15_32_32_32_32_32_32_32_32_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu: +; AVX1: # BB#0: +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,15,15,15,15,15,15,15,12,12,13,13,14,14,15,15] +; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; AVX1-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v32i8_15_15_15_15_15_15_15_15_32_32_32_32_32_32_32_32_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastb %xmm1, %xmm1 +; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,15,15,15,15,15,15,15,12,12,13,13,14,14,15,15] +; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX2-NEXT: retq + %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> + ret <32 x i8> %shuffle +} + +define <32 x i8> @shuffle_v32i8_15_15_15_15_15_15_15_15_15_15_15_15_15_15_15_15_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu(<32 x i8> %a, <32 x i8> %b) { +; ALL-LABEL: shuffle_v32i8_15_15_15_15_15_15_15_15_15_15_15_15_15_15_15_15_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu: +; ALL: # BB#0: +; ALL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; ALL-NEXT: retq + %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> + ret <32 x i8> %shuffle +} + +define <32 x i8> @shuffle_v32i8_22_22_22_22_22_22_22_22_22_22_22_22_22_22_22_22_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu(<32 x i8> %a, <32 x i8> %b) { +; AVX1-LABEL: shuffle_v32i8_22_22_22_22_22_22_22_22_22_22_22_22_22_22_22_22_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6] +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v32i8_22_22_22_22_22_22_22_22_22_22_22_22_22_22_22_22_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu: +; AVX2: # BB#0: +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 +; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6] +; AVX2-NEXT: retq + %shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> + ret <32 x i8> %shuffle +} + +define <32 x i8> @insert_dup_mem_v32i8_i32(i32* %ptr) { +; AVX1-LABEL: insert_dup_mem_v32i8_i32: +; AVX1: # BB#0: +; AVX1-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_mem_v32i8_i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastb (%rdi), %ymm0 +; AVX2-NEXT: retq + %tmp = load i32, i32* %ptr, align 4 + %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0 + %tmp2 = bitcast <4 x i32> %tmp1 to <16 x i8> + %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> undef, <32 x i32> zeroinitializer + ret <32 x i8> %tmp3 +} + +define <32 x i8> @insert_dup_mem_v32i8_sext_i8(i8* %ptr) { +; AVX1-LABEL: insert_dup_mem_v32i8_sext_i8: +; AVX1: # BB#0: +; AVX1-NEXT: movsbl (%rdi), %eax +; AVX1-NEXT: vmovd %eax, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_mem_v32i8_sext_i8: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastb (%rdi), %ymm0 +; AVX2-NEXT: retq + %tmp = load i8, i8* %ptr, align 1 + %tmp1 = sext i8 %tmp to i32 + %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0 + %tmp3 = bitcast <4 x i32> %tmp2 to <16 x i8> + %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> undef, <32 x i32> zeroinitializer + ret <32 x i8> %tmp4 +} + +define <32 x i8> @insert_dup_elt1_mem_v32i8_i32(i32* %ptr) { +; AVX1-LABEL: insert_dup_elt1_mem_v32i8_i32: +; AVX1: # BB#0: +; AVX1-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_elt1_mem_v32i8_i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastb 1(%rdi), %ymm0 +; AVX2-NEXT: retq + %tmp = load i32, i32* %ptr, align 4 + %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0 + %tmp2 = bitcast <4 x i32> %tmp1 to <16 x i8> + %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> undef, <32 x i32> + ret <32 x i8> %tmp3 +} + +define <32 x i8> @insert_dup_elt3_mem_v32i8_i32(i32* %ptr) { +; AVX1-LABEL: insert_dup_elt3_mem_v32i8_i32: +; AVX1: # BB#0: +; AVX1-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_elt3_mem_v32i8_i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastb 3(%rdi), %ymm0 +; AVX2-NEXT: retq + %tmp = load i32, i32* %ptr, align 4 + %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0 + %tmp2 = bitcast <4 x i32> %tmp1 to <16 x i8> + %tmp3 = shufflevector <16 x i8> %tmp2, <16 x i8> undef, <32 x i32> + ret <32 x i8> %tmp3 +} + +define <32 x i8> @insert_dup_elt1_mem_v32i8_sext_i8(i8* %ptr) { +; AVX1-LABEL: insert_dup_elt1_mem_v32i8_sext_i8: +; AVX1: # BB#0: +; AVX1-NEXT: movsbl (%rdi), %eax +; AVX1-NEXT: vmovd %eax, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_elt1_mem_v32i8_sext_i8: +; AVX2: # BB#0: +; AVX2-NEXT: movsbl (%rdi), %eax +; AVX2-NEXT: shrl $8, %eax +; AVX2-NEXT: vmovd %eax, %xmm0 +; AVX2-NEXT: vpbroadcastb %xmm0, %ymm0 +; AVX2-NEXT: retq + %tmp = load i8, i8* %ptr, align 1 + %tmp1 = sext i8 %tmp to i32 + %tmp2 = insertelement <4 x i32> zeroinitializer, i32 %tmp1, i32 0 + %tmp3 = bitcast <4 x i32> %tmp2 to <16 x i8> + %tmp4 = shufflevector <16 x i8> %tmp3, <16 x i8> undef, <32 x i32> + ret <32 x i8> %tmp4 +} diff --git a/test/CodeGen/X86/vector-shuffle-256-v4.ll b/test/CodeGen/X86/vector-shuffle-256-v4.ll index 62bf288a870d..7e33f5f3aa86 100644 --- a/test/CodeGen/X86/vector-shuffle-256-v4.ll +++ b/test/CodeGen/X86/vector-shuffle-256-v4.ll @@ -1,5 +1,7 @@ -; RUN: llc < %s -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 -; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX1 +; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX2 +; RUN: llc < %s -mcpu=knl -mattr=+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX512VL target triple = "x86_64-unknown-unknown" @@ -14,6 +16,11 @@ define <4 x double> @shuffle_v4f64_0000(<4 x double> %a, <4 x double> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vbroadcastsd %xmm0, %ymm0 ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4f64_0000: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vbroadcastsd %xmm0, %ymm0 +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> ret <4 x double> %shuffle } @@ -29,6 +36,11 @@ define <4 x double> @shuffle_v4f64_0001(<4 x double> %a, <4 x double> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,0,0,1] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4f64_0001: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,0,0,1] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> ret <4 x double> %shuffle } @@ -46,6 +58,11 @@ define <4 x double> @shuffle_v4f64_0020(<4 x double> %a, <4 x double> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,0,2,0] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4f64_0020: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,0,2,0] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> ret <4 x double> %shuffle } @@ -62,6 +79,11 @@ define <4 x double> @shuffle_v4f64_0300(<4 x double> %a, <4 x double> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,3,0,0] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4f64_0300: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,3,0,0] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> ret <4 x double> %shuffle } @@ -78,6 +100,11 @@ define <4 x double> @shuffle_v4f64_1000(<4 x double> %a, <4 x double> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,0,0] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4f64_1000: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,0,0] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> ret <4 x double> %shuffle } @@ -93,6 +120,11 @@ define <4 x double> @shuffle_v4f64_2200(<4 x double> %a, <4 x double> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,2,0,0] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4f64_2200: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,2,0,0] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> ret <4 x double> %shuffle } @@ -109,6 +141,11 @@ define <4 x double> @shuffle_v4f64_3330(<4 x double> %a, <4 x double> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[3,3,3,0] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4f64_3330: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[3,3,3,0] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> ret <4 x double> %shuffle } @@ -124,6 +161,11 @@ define <4 x double> @shuffle_v4f64_3210(<4 x double> %a, <4 x double> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[3,2,1,0] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4f64_3210: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[3,2,1,0] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> ret <4 x double> %shuffle } @@ -133,6 +175,7 @@ define <4 x double> @shuffle_v4f64_0023(<4 x double> %a, <4 x double> %b) { ; ALL: # BB#0: ; ALL-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,3] ; ALL-NEXT: retq + %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> ret <4 x double> %shuffle } @@ -146,6 +189,16 @@ define <4 x double> @shuffle_v4f64_0022(<4 x double> %a, <4 x double> %b) { ret <4 x double> %shuffle } +define <4 x double> @shuffle_v4f64mem_0022(<4 x double>* %ptr, <4 x double> %b) { +; ALL-LABEL: shuffle_v4f64mem_0022: +; ALL: # BB#0: +; ALL-NEXT: vmovddup {{.*#+}} ymm0 = mem[0,0,2,2] +; ALL-NEXT: retq + %a = load <4 x double>, <4 x double>* %ptr + %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> + ret <4 x double> %shuffle +} + define <4 x double> @shuffle_v4f64_1032(<4 x double> %a, <4 x double> %b) { ; ALL-LABEL: shuffle_v4f64_1032: ; ALL: # BB#0: @@ -183,17 +236,11 @@ define <4 x double> @shuffle_v4f64_1022(<4 x double> %a, <4 x double> %b) { } define <4 x double> @shuffle_v4f64_0423(<4 x double> %a, <4 x double> %b) { -; AVX1-LABEL: shuffle_v4f64_0423: -; AVX1: # BB#0: -; AVX1-NEXT: vmovddup {{.*#+}} ymm1 = ymm1[0,0,2,2] -; AVX1-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3] -; AVX1-NEXT: retq -; -; AVX2-LABEL: shuffle_v4f64_0423: -; AVX2: # BB#0: -; AVX2-NEXT: vbroadcastsd %xmm1, %ymm1 -; AVX2-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3] -; AVX2-NEXT: retq +; ALL-LABEL: shuffle_v4f64_0423: +; ALL: # BB#0: +; ALL-NEXT: vmovddup {{.*#+}} xmm1 = xmm1[0,0] +; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3] +; ALL-NEXT: retq %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> ret <4 x double> %shuffle } @@ -273,19 +320,39 @@ define <4 x double> @shuffle_v4f64_4163(<4 x double> %a, <4 x double> %b) { } define <4 x double> @shuffle_v4f64_0145(<4 x double> %a, <4 x double> %b) { -; ALL-LABEL: shuffle_v4f64_0145: -; ALL: # BB#0: -; ALL-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; ALL-NEXT: retq +; AVX1-LABEL: shuffle_v4f64_0145: +; AVX1: # BB#0: +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v4f64_0145: +; AVX2: # BB#0: +; AVX2-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4f64_0145: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vinsertf32x4 $1, %xmm1, %ymm0, %ymm0 +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> ret <4 x double> %shuffle } define <4 x double> @shuffle_v4f64_4501(<4 x double> %a, <4 x double> %b) { -; ALL-LABEL: shuffle_v4f64_4501: -; ALL: # BB#0: -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 -; ALL-NEXT: retq +; AVX1-LABEL: shuffle_v4f64_4501: +; AVX1: # BB#0: +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v4f64_4501: +; AVX2: # BB#0: +; AVX2-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4f64_4501: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vinsertf32x4 $1, %xmm0, %ymm1, %ymm0 +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> ret <4 x double> %shuffle } @@ -300,31 +367,67 @@ define <4 x double> @shuffle_v4f64_0167(<4 x double> %a, <4 x double> %b) { } define <4 x double> @shuffle_v4f64_1054(<4 x double> %a, <4 x double> %b) { -; ALL-LABEL: shuffle_v4f64_1054: -; ALL: # BB#0: -; ALL-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; ALL-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2] -; ALL-NEXT: retq +; AVX1-LABEL: shuffle_v4f64_1054: +; AVX1: # BB#0: +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2] +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v4f64_1054: +; AVX2: # BB#0: +; AVX2-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2] +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4f64_1054: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vinsertf32x4 $1, %xmm1, %ymm0, %ymm0 +; AVX512VL-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> ret <4 x double> %shuffle } define <4 x double> @shuffle_v4f64_3254(<4 x double> %a, <4 x double> %b) { -; ALL-LABEL: shuffle_v4f64_3254: -; ALL: # BB#0: -; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] -; ALL-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2] -; ALL-NEXT: retq +; AVX1-LABEL: shuffle_v4f64_3254: +; AVX1: # BB#0: +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] +; AVX1-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2] +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v4f64_3254: +; AVX2: # BB#0: +; AVX2-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] +; AVX2-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2] +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4f64_3254: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] +; AVX512VL-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> ret <4 x double> %shuffle } define <4 x double> @shuffle_v4f64_3276(<4 x double> %a, <4 x double> %b) { -; ALL-LABEL: shuffle_v4f64_3276: -; ALL: # BB#0: -; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3] -; ALL-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2] -; ALL-NEXT: retq +; AVX1-LABEL: shuffle_v4f64_3276: +; AVX1: # BB#0: +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3] +; AVX1-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2] +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v4f64_3276: +; AVX2: # BB#0: +; AVX2-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3] +; AVX2-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2] +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4f64_3276: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3] +; AVX512VL-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> ret <4 x double> %shuffle } @@ -353,6 +456,13 @@ define <4 x double> @shuffle_v4f64_0415(<4 x double> %a, <4 x double> %b) { ; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,3] ; AVX2-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4f64_0415: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,0,2,1] +; AVX512VL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,3] +; AVX512VL-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> ret <4 x double> %shuffle } @@ -366,6 +476,65 @@ define <4 x double> @shuffle_v4f64_u062(<4 x double> %a, <4 x double> %b) { ret <4 x double> %shuffle } +define <4 x double> @shuffle_v4f64_15uu(<4 x double> %a, <4 x double> %b) { +; ALL-LABEL: shuffle_v4f64_15uu: +; ALL: # BB#0: +; ALL-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] +; ALL-NEXT: retq + %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> + ret <4 x double> %shuffle +} + +define <4 x double> @shuffle_v4f64_11uu(<4 x double> %a, <4 x double> %b) { +; ALL-LABEL: shuffle_v4f64_11uu: +; ALL: # BB#0: +; ALL-NEXT: vmovhlps {{.*#+}} xmm0 = xmm0[1,1] +; ALL-NEXT: retq + %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> + ret <4 x double> %shuffle +} + +define <4 x double> @shuffle_v4f64_22uu(<4 x double> %a, <4 x double> %b) { +; AVX1-LABEL: shuffle_v4f64_22uu: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0] +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v4f64_22uu: +; AVX2: # BB#0: +; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,2,2,3] +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4f64_22uu: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,2,2,3] +; AVX512VL-NEXT: retq + %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> + ret <4 x double> %shuffle +} + +define <4 x double> @shuffle_v4f64_3333(<4 x double> %a, <4 x double> %b) { +; AVX1-LABEL: shuffle_v4f64_3333: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vmovhlps {{.*#+}} xmm0 = xmm0[1,1] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v4f64_3333: +; AVX2: # BB#0: +; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[3,3,3,3] +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4f64_3333: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[3,3,3,3] +; AVX512VL-NEXT: retq + %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> + ret <4 x double> %shuffle +} + define <4 x i64> @shuffle_v4i64_0000(<4 x i64> %a, <4 x i64> %b) { ; AVX1-LABEL: shuffle_v4i64_0000: ; AVX1: # BB#0: @@ -377,6 +546,11 @@ define <4 x i64> @shuffle_v4i64_0000(<4 x i64> %a, <4 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vbroadcastsd %xmm0, %ymm0 ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_0000: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpbroadcastq %xmm0, %ymm0 +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -392,6 +566,11 @@ define <4 x i64> @shuffle_v4i64_0001(<4 x i64> %a, <4 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,0,1] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_0001: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,0,1] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -409,6 +588,11 @@ define <4 x i64> @shuffle_v4i64_0020(<4 x i64> %a, <4 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,0] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_0020: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,0] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -425,6 +609,11 @@ define <4 x i64> @shuffle_v4i64_0112(<4 x i64> %a, <4 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,1,2] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_0112: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,1,2] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -441,6 +630,11 @@ define <4 x i64> @shuffle_v4i64_0300(<4 x i64> %a, <4 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,0,0] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_0300: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,0,0] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -457,6 +651,11 @@ define <4 x i64> @shuffle_v4i64_1000(<4 x i64> %a, <4 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,0,0,0] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_1000: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,0,0,0] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -472,6 +671,11 @@ define <4 x i64> @shuffle_v4i64_2200(<4 x i64> %a, <4 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,0,0] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_2200: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,0,0] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -488,6 +692,11 @@ define <4 x i64> @shuffle_v4i64_3330(<4 x i64> %a, <4 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,3,3,0] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_3330: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,3,3,0] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -503,6 +712,11 @@ define <4 x i64> @shuffle_v4i64_3210(<4 x i64> %a, <4 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_3210: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -520,6 +734,12 @@ define <4 x i64> @shuffle_v4i64_0124(<4 x i64> %a, <4 x i64> %b) { ; AVX2-NEXT: vpbroadcastq %xmm1, %ymm1 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_0124: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpbroadcastq %xmm1, %ymm1 +; AVX512VL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -527,17 +747,24 @@ define <4 x i64> @shuffle_v4i64_0124(<4 x i64> %a, <4 x i64> %b) { define <4 x i64> @shuffle_v4i64_0142(<4 x i64> %a, <4 x i64> %b) { ; AVX1-LABEL: shuffle_v4i64_0142: ; AVX1: # BB#0: -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm1, %ymm1 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 ; AVX1-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,1,2,2] ; AVX1-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3] ; AVX1-NEXT: retq ; ; AVX2-LABEL: shuffle_v4i64_0142: ; AVX2: # BB#0: -; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm1, %ymm1 +; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,2,2] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_0142: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vinserti32x4 $1, %xmm1, %ymm0, %ymm1 +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,2,2] +; AVX512VL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -548,16 +775,23 @@ define <4 x i64> @shuffle_v4i64_0412(<4 x i64> %a, <4 x i64> %b) { ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 ; AVX1-NEXT: vshufpd {{.*#+}} xmm2 = xmm0[1],xmm2[0] ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 -; AVX1-NEXT: vmovddup {{.*#+}} ymm1 = ymm1[0,0,2,2] +; AVX1-NEXT: vmovddup {{.*#+}} xmm1 = xmm1[0,0] ; AVX1-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3] ; AVX1-NEXT: retq ; ; AVX2-LABEL: shuffle_v4i64_0412: ; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastq %xmm1, %xmm1 ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,1,2] -; AVX2-NEXT: vpbroadcastq %xmm1, %ymm1 ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_0412: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpbroadcastq %xmm1, %xmm1 +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,1,2] +; AVX512VL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -577,15 +811,31 @@ define <4 x i64> @shuffle_v4i64_4012(<4 x i64> %a, <4 x i64> %b) { ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,1,2] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_4012: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,1,2] +; AVX512VL-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } define <4 x i64> @shuffle_v4i64_0145(<4 x i64> %a, <4 x i64> %b) { -; ALL-LABEL: shuffle_v4i64_0145: -; ALL: # BB#0: -; ALL-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; ALL-NEXT: retq +; AVX1-LABEL: shuffle_v4i64_0145: +; AVX1: # BB#0: +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v4i64_0145: +; AVX2: # BB#0: +; AVX2-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_0145: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vinserti32x4 $1, %xmm1, %ymm0, %ymm0 +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -604,15 +854,32 @@ define <4 x i64> @shuffle_v4i64_0451(<4 x i64> %a, <4 x i64> %b) { ; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,1,3] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3,4,5],ymm0[6,7] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_0451: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vinserti32x4 $1, %xmm0, %ymm0, %ymm0 +; AVX512VL-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,1,3] +; AVX512VL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3,4,5],ymm0[6,7] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } define <4 x i64> @shuffle_v4i64_4501(<4 x i64> %a, <4 x i64> %b) { -; ALL-LABEL: shuffle_v4i64_4501: -; ALL: # BB#0: -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 -; ALL-NEXT: retq +; AVX1-LABEL: shuffle_v4i64_4501: +; AVX1: # BB#0: +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v4i64_4501: +; AVX2: # BB#0: +; AVX2-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_4501: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vinserti32x4 $1, %xmm0, %ymm1, %ymm0 +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -631,6 +898,13 @@ define <4 x i64> @shuffle_v4i64_4015(<4 x i64> %a, <4 x i64> %b) { ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,1,3] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5],ymm1[6,7] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_4015: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vinserti32x4 $1, %xmm1, %ymm1, %ymm1 +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,1,3] +; AVX512VL-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5],ymm1[6,7] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -648,6 +922,12 @@ define <4 x i64> @shuffle_v4i64_2u35(<4 x i64> %a, <4 x i64> %b) { ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7] ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,1,3,1] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_2u35: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7] +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,1,3,1] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -668,6 +948,13 @@ define <4 x i64> @shuffle_v4i64_1251(<4 x i64> %a, <4 x i64> %b) { ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,2,2,1] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_1251: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3] +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,2,2,1] +; AVX512VL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -684,6 +971,12 @@ define <4 x i64> @shuffle_v4i64_1054(<4 x i64> %a, <4 x i64> %b) { ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,3,0,1,6,7,4,5] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_1054: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vinserti32x4 $1, %xmm1, %ymm0, %ymm0 +; AVX512VL-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,3,0,1,6,7,4,5] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -700,6 +993,12 @@ define <4 x i64> @shuffle_v4i64_3254(<4 x i64> %a, <4 x i64> %b) { ; AVX2-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] ; AVX2-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,3,0,1,6,7,4,5] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_3254: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] +; AVX512VL-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,3,0,1,6,7,4,5] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -716,6 +1015,12 @@ define <4 x i64> @shuffle_v4i64_3276(<4 x i64> %a, <4 x i64> %b) { ; AVX2-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3] ; AVX2-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,3,0,1,6,7,4,5] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_3276: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3] +; AVX512VL-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,3,0,1,6,7,4,5] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -732,6 +1037,12 @@ define <4 x i64> @shuffle_v4i64_1076(<4 x i64> %a, <4 x i64> %b) { ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; AVX2-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,3,0,1,6,7,4,5] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_1076: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; AVX512VL-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,3,0,1,6,7,4,5] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -750,6 +1061,13 @@ define <4 x i64> @shuffle_v4i64_0415(<4 x i64> %a, <4 x i64> %b) { ; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3] ; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_0415: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1] +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3] +; AVX512VL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } @@ -765,6 +1083,11 @@ define <4 x i64> @shuffle_v4i64_z4z6(<4 x i64> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpslldq {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,ymm0[0,1,2,3,4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,18,19,20,21,22,23] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_z4z6: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpslldq {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,ymm0[0,1,2,3,4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,18,19,20,21,22,23] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> zeroinitializer, <4 x i64> %a, <4 x i32> ret <4 x i64> %shuffle } @@ -780,6 +1103,11 @@ define <4 x i64> @shuffle_v4i64_5zuz(<4 x i64> %a) { ; AVX2: # BB#0: ; AVX2-NEXT: vpsrldq {{.*#+}} ymm0 = ymm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_5zuz: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpsrldq {{.*#+}} ymm0 = ymm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> zeroinitializer, <4 x i64> %a, <4 x i32> ret <4 x i64> %shuffle } @@ -794,10 +1122,74 @@ define <4 x i64> @shuffle_v4i64_40u2(<4 x i64> %a, <4 x i64> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[2],ymm0[2] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_40u2: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[2],ymm0[2] +; AVX512VL-NEXT: retq %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> ret <4 x i64> %shuffle } +define <4 x i64> @shuffle_v4i64_15uu(<4 x i64> %a, <4 x i64> %b) { +; ALL-LABEL: shuffle_v4i64_15uu: +; ALL: # BB#0: +; ALL-NEXT: vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] +; ALL-NEXT: retq + %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> + ret <4 x i64> %shuffle +} + +define <4 x i64> @shuffle_v4i64_11uu(<4 x i64> %a, <4 x i64> %b) { +; ALL-LABEL: shuffle_v4i64_11uu: +; ALL: # BB#0: +; ALL-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] +; ALL-NEXT: retq + %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> + ret <4 x i64> %shuffle +} + +define <4 x i64> @shuffle_v4i64_22uu(<4 x i64> %a, <4 x i64> %b) { +; AVX1-LABEL: shuffle_v4i64_22uu: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0] +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v4i64_22uu: +; AVX2: # BB#0: +; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3] +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_22uu: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3] +; AVX512VL-NEXT: retq + %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> + ret <4 x i64> %shuffle +} + +define <4 x i64> @shuffle_v4i64_3333(<4 x i64> %a, <4 x i64> %b) { +; AVX1-LABEL: shuffle_v4i64_3333: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vpermilpd {{.*#+}} xmm0 = xmm0[1,1] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v4i64_3333: +; AVX2: # BB#0: +; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,3,3,3] +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: shuffle_v4i64_3333: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,3,3,3] +; AVX512VL-NEXT: retq + %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> + ret <4 x i64> %shuffle +} + define <4 x i64> @stress_test1(<4 x i64> %a, <4 x i64> %b) { ; ALL-LABEL: stress_test1: ; ALL: retq @@ -820,10 +1212,20 @@ define <4 x i64> @insert_reg_and_zero_v4i64(i64 %a) { } define <4 x i64> @insert_mem_and_zero_v4i64(i64* %ptr) { -; ALL-LABEL: insert_mem_and_zero_v4i64: -; ALL: # BB#0: -; ALL-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero -; ALL-NEXT: retq +; AVX1-LABEL: insert_mem_and_zero_v4i64: +; AVX1: # BB#0: +; AVX1-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_mem_and_zero_v4i64: +; AVX2: # BB#0: +; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: insert_mem_and_zero_v4i64: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vmovq (%rdi), %xmm0 +; AVX512VL-NEXT: retq %a = load i64, i64* %ptr %v = insertelement <4 x i64> undef, i64 %a, i64 0 %shuffle = shufflevector <4 x i64> %v, <4 x i64> zeroinitializer, <4 x i32> @@ -831,21 +1233,43 @@ define <4 x i64> @insert_mem_and_zero_v4i64(i64* %ptr) { } define <4 x double> @insert_reg_and_zero_v4f64(double %a) { -; ALL-LABEL: insert_reg_and_zero_v4f64: -; ALL: # BB#0: -; ALL-NEXT: vxorpd %ymm1, %ymm1, %ymm1 -; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3] -; ALL-NEXT: retq +; AVX1-LABEL: insert_reg_and_zero_v4f64: +; AVX1: # BB#0: +; AVX1-NEXT: vxorpd %ymm1, %ymm1, %ymm1 +; AVX1-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3] +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_reg_and_zero_v4f64: +; AVX2: # BB#0: +; AVX2-NEXT: vxorpd %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3] +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: insert_reg_and_zero_v4f64: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vxorpd %xmm1, %xmm1, %xmm1 +; AVX512VL-NEXT: vmovsd %xmm0, %xmm1, %xmm0 +; AVX512VL-NEXT: retq %v = insertelement <4 x double> undef, double %a, i32 0 %shuffle = shufflevector <4 x double> %v, <4 x double> zeroinitializer, <4 x i32> ret <4 x double> %shuffle } define <4 x double> @insert_mem_and_zero_v4f64(double* %ptr) { -; ALL-LABEL: insert_mem_and_zero_v4f64: -; ALL: # BB#0: -; ALL-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero -; ALL-NEXT: retq +; AVX1-LABEL: insert_mem_and_zero_v4f64: +; AVX1: # BB#0: +; AVX1-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_mem_and_zero_v4f64: +; AVX2: # BB#0: +; AVX2-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: insert_mem_and_zero_v4f64: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vmovsd (%rdi), %xmm0 +; AVX512VL-NEXT: retq %a = load double, double* %ptr %v = insertelement <4 x double> undef, double %a, i32 0 %shuffle = shufflevector <4 x double> %v, <4 x double> zeroinitializer, <4 x i32> @@ -864,10 +1288,20 @@ define <4 x double> @splat_mem_v4f64(double* %ptr) { } define <4 x i64> @splat_mem_v4i64(i64* %ptr) { -; ALL-LABEL: splat_mem_v4i64: -; ALL: # BB#0: -; ALL-NEXT: vbroadcastsd (%rdi), %ymm0 -; ALL-NEXT: retq +; AVX1-LABEL: splat_mem_v4i64: +; AVX1: # BB#0: +; AVX1-NEXT: vbroadcastsd (%rdi), %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: splat_mem_v4i64: +; AVX2: # BB#0: +; AVX2-NEXT: vbroadcastsd (%rdi), %ymm0 +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: splat_mem_v4i64: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpbroadcastq (%rdi), %ymm0 +; AVX512VL-NEXT: retq %a = load i64, i64* %ptr %v = insertelement <4 x i64> undef, i64 %a, i64 0 %shuffle = shufflevector <4 x i64> %v, <4 x i64> undef, <4 x i32> @@ -896,6 +1330,11 @@ define <4 x double> @splat_v4f64(<2 x double> %r) { ; AVX2: # BB#0: ; AVX2-NEXT: vbroadcastsd %xmm0, %ymm0 ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: splat_v4f64: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vbroadcastsd %xmm0, %ymm0 +; AVX512VL-NEXT: retq %1 = shufflevector <2 x double> %r, <2 x double> undef, <4 x i32> zeroinitializer ret <4 x double> %1 } @@ -911,44 +1350,67 @@ define <4 x i64> @splat_mem_v4i64_from_v2i64(<2 x i64>* %ptr) { ; AVX2: # BB#0: ; AVX2-NEXT: vbroadcastsd (%rdi), %ymm0 ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: splat_mem_v4i64_from_v2i64: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpbroadcastq (%rdi), %ymm0 +; AVX512VL-NEXT: retq %v = load <2 x i64>, <2 x i64>* %ptr %shuffle = shufflevector <2 x i64> %v, <2 x i64> undef, <4 x i32> ret <4 x i64> %shuffle } define <4 x double> @splat_mem_v4f64_from_v2f64(<2 x double>* %ptr) { -; AVX1-LABEL: splat_mem_v4f64_from_v2f64: -; AVX1: # BB#0: -; AVX1-NEXT: vmovddup {{.*#+}} xmm0 = mem[0,0] -; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX1-NEXT: retq -; -; AVX2-LABEL: splat_mem_v4f64_from_v2f64: -; AVX2: # BB#0: -; AVX2-NEXT: vbroadcastsd (%rdi), %ymm0 -; AVX2-NEXT: retq +; ALL-LABEL: splat_mem_v4f64_from_v2f64: +; ALL: # BB#0: +; ALL-NEXT: vbroadcastsd (%rdi), %ymm0 +; ALL-NEXT: retq %v = load <2 x double>, <2 x double>* %ptr %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <4 x i32> ret <4 x double> %shuffle } define <4 x i64> @splat128_mem_v4i64_from_v2i64(<2 x i64>* %ptr) { -; ALL-LABEL: splat128_mem_v4i64_from_v2i64: -; ALL: # BB#0: -; ALL-NEXT: vmovaps (%rdi), %xmm0 -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; ALL-NEXT: retq +; AVX1-LABEL: splat128_mem_v4i64_from_v2i64: +; AVX1: # BB#0: +; AVX1-NEXT: vmovaps (%rdi), %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: splat128_mem_v4i64_from_v2i64: +; AVX2: # BB#0: +; AVX2-NEXT: vmovaps (%rdi), %xmm0 +; AVX2-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: splat128_mem_v4i64_from_v2i64: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vmovdqa64 (%rdi), %xmm0 +; AVX512VL-NEXT: vinserti32x4 $1, %xmm0, %ymm0, %ymm0 +; AVX512VL-NEXT: retq %v = load <2 x i64>, <2 x i64>* %ptr %shuffle = shufflevector <2 x i64> %v, <2 x i64> undef, <4 x i32> ret <4 x i64> %shuffle } define <4 x double> @splat128_mem_v4f64_from_v2f64(<2 x double>* %ptr) { -; ALL-LABEL: splat128_mem_v4f64_from_v2f64: -; ALL: # BB#0: -; ALL-NEXT: vmovaps (%rdi), %xmm0 -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; ALL-NEXT: retq +; AVX1-LABEL: splat128_mem_v4f64_from_v2f64: +; AVX1: # BB#0: +; AVX1-NEXT: vmovaps (%rdi), %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: splat128_mem_v4f64_from_v2f64: +; AVX2: # BB#0: +; AVX2-NEXT: vmovaps (%rdi), %xmm0 +; AVX2-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: splat128_mem_v4f64_from_v2f64: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vmovapd (%rdi), %xmm0 +; AVX512VL-NEXT: vinsertf32x4 $1, %xmm0, %ymm0, %ymm0 +; AVX512VL-NEXT: retq %v = load <2 x double>, <2 x double>* %ptr %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <4 x i32> ret <4 x double> %shuffle @@ -964,6 +1426,11 @@ define <4 x double> @bitcast_v4f64_0426(<4 x double> %a, <4 x double> %b) { ; AVX2: # BB#0: ; AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] ; AVX2-NEXT: retq +; +; AVX512VL-LABEL: bitcast_v4f64_0426: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] +; AVX512VL-NEXT: retq %shuffle64 = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> %bitcast32 = bitcast <4 x double> %shuffle64 to <8 x float> %shuffle32 = shufflevector <8 x float> %bitcast32, <8 x float> undef, <8 x i32> @@ -972,3 +1439,69 @@ define <4 x double> @bitcast_v4f64_0426(<4 x double> %a, <4 x double> %b) { %bitcast64 = bitcast <16 x i16> %shuffle16 to <4 x double> ret <4 x double> %bitcast64 } + +define <4 x i64> @concat_v4i64_0167(<4 x i64> %a0, <4 x i64> %a1) { +; AVX1-LABEL: concat_v4i64_0167: +; AVX1: # BB#0: +; AVX1-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3] +; AVX1-NEXT: retq +; +; AVX2-LABEL: concat_v4i64_0167: +; AVX2: # BB#0: +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: concat_v4i64_0167: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; AVX512VL-NEXT: retq + %a0lo = shufflevector <4 x i64> %a0, <4 x i64> %a1, <2 x i32> + %a1hi = shufflevector <4 x i64> %a0, <4 x i64> %a1, <2 x i32> + %shuffle64 = shufflevector <2 x i64> %a0lo, <2 x i64> %a1hi, <4 x i32> + ret <4 x i64> %shuffle64 +} + +define <4 x i64> @concat_v4i64_0145_bc(<4 x i64> %a0, <4 x i64> %a1) { +; AVX1-LABEL: concat_v4i64_0145_bc: +; AVX1: # BB#0: +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: concat_v4i64_0145_bc: +; AVX2: # BB#0: +; AVX2-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: concat_v4i64_0145_bc: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vinserti32x4 $1, %xmm1, %ymm0, %ymm0 +; AVX512VL-NEXT: retq + %a0lo = shufflevector <4 x i64> %a0, <4 x i64> %a1, <2 x i32> + %a1lo = shufflevector <4 x i64> %a0, <4 x i64> %a1, <2 x i32> + %bc0lo = bitcast <2 x i64> %a0lo to <4 x i32> + %bc1lo = bitcast <2 x i64> %a1lo to <4 x i32> + %shuffle32 = shufflevector <4 x i32> %bc0lo, <4 x i32> %bc1lo, <8 x i32> + %shuffle64 = bitcast <8 x i32> %shuffle32 to <4 x i64> + ret <4 x i64> %shuffle64 +} + +define <4 x i64> @insert_dup_mem_v4i64(i64* %ptr) { +; AVX1-LABEL: insert_dup_mem_v4i64: +; AVX1: # BB#0: +; AVX1-NEXT: vbroadcastsd (%rdi), %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: insert_dup_mem_v4i64: +; AVX2: # BB#0: +; AVX2-NEXT: vbroadcastsd (%rdi), %ymm0 +; AVX2-NEXT: retq +; +; AVX512VL-LABEL: insert_dup_mem_v4i64: +; AVX512VL: # BB#0: +; AVX512VL-NEXT: vpbroadcastq (%rdi), %ymm0 +; AVX512VL-NEXT: retq + %tmp = load i64, i64* %ptr, align 1 + %tmp1 = insertelement <2 x i64> undef, i64 %tmp, i32 0 + %tmp2 = shufflevector <2 x i64> %tmp1, <2 x i64> undef, <4 x i32> zeroinitializer + ret <4 x i64> %tmp2 +} diff --git a/test/CodeGen/X86/vector-shuffle-256-v8.ll b/test/CodeGen/X86/vector-shuffle-256-v8.ll index bc72e0a66177..e8b886afd1ae 100644 --- a/test/CodeGen/X86/vector-shuffle-256-v8.ll +++ b/test/CodeGen/X86/vector-shuffle-256-v8.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX1 ; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX2 @@ -72,10 +73,10 @@ define <8 x float> @shuffle_v8f32_00003000(<8 x float> %a, <8 x float> %b) { define <8 x float> @shuffle_v8f32_00040000(<8 x float> %a, <8 x float> %b) { ; AVX1-LABEL: shuffle_v8f32_00040000: ; AVX1: # BB#0: -; AVX1-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm0[2,3,0,1] -; AVX1-NEXT: vpermilps {{.*#+}} ymm1 = ymm1[0,0,0,0,4,4,4,4] -; AVX1-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,3,4,4,4,7] -; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7] +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[0,0,0,3] +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1] +; AVX1-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4] +; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7] ; AVX1-NEXT: retq ; ; AVX2-LABEL: shuffle_v8f32_00040000: @@ -830,6 +831,87 @@ define <8 x float> @shuffle_v8f32_a2u3e6f7(<8 x float> %a, <8 x float> %b) { ret <8 x float> %shuffle } +define <8 x float> @shuffle_v8f32_uuuu1111(<8 x float> %a, <8 x float> %b) { +; ALL-LABEL: shuffle_v8f32_uuuu1111: +; ALL: # BB#0: +; ALL-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[1,1,1,1] +; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; ALL-NEXT: retq + %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> + ret <8 x float> %shuffle +} + +define <8 x float> @shuffle_v8f32_44444444(<8 x float> %a, <8 x float> %b) { +; AVX1-LABEL: shuffle_v8f32_44444444: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[0,0,0,0] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v8f32_44444444: +; AVX2: # BB#0: +; AVX2-NEXT: vbroadcastss {{.*}}(%rip), %ymm1 +; AVX2-NEXT: vpermps %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq + %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> + ret <8 x float> %shuffle +} + +define <8 x float> @shuffle_v8f32_1188uuuu(<8 x float> %a, <8 x float> %b) { +; ALL-LABEL: shuffle_v8f32_1188uuuu: +; ALL: # BB#0: +; ALL-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[0,0] +; ALL-NEXT: retq + %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> + ret <8 x float> %shuffle +} + +define <8 x float> @shuffle_v8f32_uuuu3210(<8 x float> %a, <8 x float> %b) { +; ALL-LABEL: shuffle_v8f32_uuuu3210: +; ALL: # BB#0: +; ALL-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[3,2,1,0] +; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; ALL-NEXT: retq + %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> + ret <8 x float> %shuffle +} + +define <8 x float> @shuffle_v8f32_uuuu1188(<8 x float> %a, <8 x float> %b) { +; ALL-LABEL: shuffle_v8f32_uuuu1188: +; ALL: # BB#0: +; ALL-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[0,0] +; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; ALL-NEXT: retq + %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> + ret <8 x float> %shuffle +} + +define <8 x float> @shuffle_v8f32_1111uuuu(<8 x float> %a, <8 x float> %b) { +; ALL-LABEL: shuffle_v8f32_1111uuuu: +; ALL: # BB#0: +; ALL-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[1,1,1,1] +; ALL-NEXT: retq + %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> + ret <8 x float> %shuffle +} + +define <8 x float> @shuffle_v8f32_5555uuuu(<8 x float> %a, <8 x float> %b) { +; AVX1-LABEL: shuffle_v8f32_5555uuuu: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[1,1,1,1] +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v8f32_5555uuuu: +; AVX2: # BB#0: +; AVX2-NEXT: vbroadcastss {{.*}}(%rip), %ymm1 +; AVX2-NEXT: vpermps %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq + %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> + ret <8 x float> %shuffle +} + define <8 x i32> @shuffle_v8i32_00000000(<8 x i32> %a, <8 x i32> %b) { ; AVX1-LABEL: shuffle_v8i32_00000000: ; AVX1: # BB#0: @@ -899,10 +981,10 @@ define <8 x i32> @shuffle_v8i32_00003000(<8 x i32> %a, <8 x i32> %b) { define <8 x i32> @shuffle_v8i32_00040000(<8 x i32> %a, <8 x i32> %b) { ; AVX1-LABEL: shuffle_v8i32_00040000: ; AVX1: # BB#0: -; AVX1-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm0[2,3,0,1] -; AVX1-NEXT: vpermilps {{.*#+}} ymm1 = ymm1[0,0,0,0,4,4,4,4] -; AVX1-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,3,4,4,4,7] -; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7] +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[0,0,0,3] +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1] +; AVX1-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4] +; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7] ; AVX1-NEXT: retq ; ; AVX2-LABEL: shuffle_v8i32_00040000: @@ -1895,6 +1977,73 @@ define <8 x i32> @shuffle_v8i32_80u1b4uu(<8 x i32> %a, <8 x i32> %b) { ret <8 x i32> %shuffle } +define <8 x i32> @shuffle_v8i32_uuuu1111(<8 x i32> %a, <8 x i32> %b) { +; AVX1-LABEL: shuffle_v8i32_uuuu1111: +; AVX1: # BB#0: +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v8i32_uuuu1111: +; AVX2: # BB#0: +; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1] +; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 +; AVX2-NEXT: retq + %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> + ret <8 x i32> %shuffle +} + +define <8 x i32> @shuffle_v8i32_2222uuuu(<8 x i32> %a, <8 x i32> %b) { +; ALL-LABEL: shuffle_v8i32_2222uuuu: +; ALL: # BB#0: +; ALL-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,2,2,2] +; ALL-NEXT: retq + %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> + ret <8 x i32> %shuffle +} + +define <8 x i32> @shuffle_v8i32_2A3Buuuu(<8 x i32> %a, <8 x i32> %b) { +; ALL-LABEL: shuffle_v8i32_2A3Buuuu: +; ALL: # BB#0: +; ALL-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; ALL-NEXT: retq + %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> + ret <8 x i32> %shuffle +} + +define <8 x i32> @shuffle_v8i32_44444444(<8 x i32> %a, <8 x i32> %b) { +; AVX1-LABEL: shuffle_v8i32_44444444: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[0,0,0,0] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v8i32_44444444: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %ymm1 +; AVX2-NEXT: vpermd %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq + %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> + ret <8 x i32> %shuffle +} + +define <8 x i32> @shuffle_v8i32_5555uuuu(<8 x i32> %a, <8 x i32> %b) { +; AVX1-LABEL: shuffle_v8i32_5555uuuu: +; AVX1: # BB#0: +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[1,1,1,1] +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuffle_v8i32_5555uuuu: +; AVX2: # BB#0: +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %ymm1 +; AVX2-NEXT: vpermd %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq + %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> + ret <8 x i32> %shuffle +} + define <8 x float> @splat_mem_v8f32_2(float* %p) { ; ALL-LABEL: splat_mem_v8f32_2: ; ALL: # BB#0: @@ -2098,3 +2247,59 @@ define <8 x i32> @insert_mem_and_zero_v8i32(i32* %ptr) { ret <8 x i32> %shuffle } +define <8 x i32> @concat_v8i32_0123CDEF(<8 x i32> %a, <8 x i32> %b) { +; AVX1-LABEL: concat_v8i32_0123CDEF: +; AVX1: # BB#0: +; AVX1-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3] +; AVX1-NEXT: retq +; +; AVX2-LABEL: concat_v8i32_0123CDEF: +; AVX2: # BB#0: +; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; AVX2-NEXT: retq + %alo = shufflevector <8 x i32> %a, <8 x i32> undef, <4 x i32> + %bhi = shufflevector <8 x i32> %b, <8 x i32> undef, <4 x i32> + %shuf = shufflevector <4 x i32> %alo, <4 x i32> %bhi, <8 x i32> + ret <8 x i32> %shuf +} + +define <8 x i32> @concat_v8i32_4567CDEF_bc(<8 x i32> %a0, <8 x i32> %a1) { +; ALL-LABEL: concat_v8i32_4567CDEF_bc: +; ALL: # BB#0: +; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3] +; ALL-NEXT: retq + %a0hi = shufflevector <8 x i32> %a0, <8 x i32> %a1, <4 x i32> + %a1hi = shufflevector <8 x i32> %a0, <8 x i32> %a1, <4 x i32> + %bc0hi = bitcast <4 x i32> %a0hi to <2 x i64> + %bc1hi = bitcast <4 x i32> %a1hi to <2 x i64> + %shuffle64 = shufflevector <2 x i64> %bc0hi, <2 x i64> %bc1hi, <4 x i32> + %shuffle32 = bitcast <4 x i64> %shuffle64 to <8 x i32> + ret <8 x i32> %shuffle32 +} + +define <8 x float> @concat_v8f32_4567CDEF_bc(<8 x float> %f0, <8 x float> %f1) { +; ALL-LABEL: concat_v8f32_4567CDEF_bc: +; ALL: # BB#0: +; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3] +; ALL-NEXT: retq + %a0 = bitcast <8 x float> %f0 to <4 x i64> + %a1 = bitcast <8 x float> %f1 to <8 x i32> + %a0hi = shufflevector <4 x i64> %a0, <4 x i64> undef, <2 x i32> + %a1hi = shufflevector <8 x i32> %a1, <8 x i32> undef, <4 x i32> + %bc0hi = bitcast <2 x i64> %a0hi to <2 x i64> + %bc1hi = bitcast <4 x i32> %a1hi to <2 x i64> + %shuffle64 = shufflevector <2 x i64> %bc0hi, <2 x i64> %bc1hi, <4 x i32> + %shuffle32 = bitcast <4 x i64> %shuffle64 to <8 x float> + ret <8 x float> %shuffle32 +} + +define <8 x i32> @insert_dup_mem_v8i32(i32* %ptr) { +; ALL-LABEL: insert_dup_mem_v8i32: +; ALL: # BB#0: +; ALL-NEXT: vbroadcastss (%rdi), %ymm0 +; ALL-NEXT: retq + %tmp = load i32, i32* %ptr, align 4 + %tmp1 = insertelement <4 x i32> zeroinitializer, i32 %tmp, i32 0 + %tmp2 = shufflevector <4 x i32> %tmp1, <4 x i32> undef, <8 x i32> zeroinitializer + ret <8 x i32> %tmp2 +} diff --git a/test/CodeGen/X86/vector-shuffle-512-v16.ll b/test/CodeGen/X86/vector-shuffle-512-v16.ll index 406d52406d95..bef54b05041b 100644 --- a/test/CodeGen/X86/vector-shuffle-512-v16.ll +++ b/test/CodeGen/X86/vector-shuffle-512-v16.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mcpu=x86-64 -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512F ; RUN: llc < %s -mcpu=x86-64 -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW @@ -12,6 +13,25 @@ define <16 x float> @shuffle_v16f32_00_10_01_11_04_14_05_15_08_18_09_19_0c_1c_0d ret <16 x float> %shuffle } +define <16 x float> @shuffle_v16f32_00_zz_01_zz_04_zz_05_zz_08_zz_09_zz_0c_zz_0d_zz(<16 x float> %a, <16 x float> %b) { +; ALL-LABEL: shuffle_v16f32_00_zz_01_zz_04_zz_05_zz_08_zz_09_zz_0c_zz_0d_zz: +; ALL: # BB#0: +; ALL-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; ALL-NEXT: vunpcklps {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] +; ALL-NEXT: retq + %shuffle = shufflevector <16 x float> %a, <16 x float> zeroinitializer, <16 x i32> + ret <16 x float> %shuffle +} + +define <16 x float> @shuffle_v16f32_vunpcklps_swap(<16 x float> %a, <16 x float> %b) { +; ALL-LABEL: shuffle_v16f32_vunpcklps_swap: +; ALL: # BB#0: +; ALL-NEXT: vunpcklps {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[12],zmm0[12],zmm1[13],zmm0[13] +; ALL-NEXT: retq + %shuffle = shufflevector <16 x float> %a, <16 x float> %b, <16 x i32> + ret <16 x float> %shuffle +} + define <16 x i32> @shuffle_v16i32_00_10_01_11_04_14_05_15_08_18_09_19_0c_1c_0d_1d(<16 x i32> %a, <16 x i32> %b) { ; ALL-LABEL: shuffle_v16i32_00_10_01_11_04_14_05_15_08_18_09_19_0c_1c_0d_1d: ; ALL: # BB#0: @@ -21,6 +41,16 @@ define <16 x i32> @shuffle_v16i32_00_10_01_11_04_14_05_15_08_18_09_19_0c_1c_0d_1 ret <16 x i32> %shuffle } +define <16 x i32> @shuffle_v16i32_zz_10_zz_11_zz_14_zz_15_zz_18_zz_19_zz_1c_zz_1d(<16 x i32> %a, <16 x i32> %b) { +; ALL-LABEL: shuffle_v16i32_zz_10_zz_11_zz_14_zz_15_zz_18_zz_19_zz_1c_zz_1d: +; ALL: # BB#0: +; ALL-NEXT: vpxord %zmm0, %zmm0, %zmm0 +; ALL-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] +; ALL-NEXT: retq + %shuffle = shufflevector <16 x i32> zeroinitializer, <16 x i32> %b, <16 x i32> + ret <16 x i32> %shuffle +} + define <16 x float> @shuffle_v16f32_02_12_03_13_06_16_07_17_0a_1a_0b_1b_0e_1e_0f_1f(<16 x float> %a, <16 x float> %b) { ; ALL-LABEL: shuffle_v16f32_02_12_03_13_06_16_07_17_0a_1a_0b_1b_0e_1e_0f_1f: ; ALL: # BB#0: @@ -30,6 +60,16 @@ define <16 x float> @shuffle_v16f32_02_12_03_13_06_16_07_17_0a_1a_0b_1b_0e_1e_0f ret <16 x float> %shuffle } +define <16 x float> @shuffle_v16f32_zz_12_zz_13_zz_16_zz_17_zz_1a_zz_1b_zz_1e_zz_1f(<16 x float> %a, <16 x float> %b) { +; ALL-LABEL: shuffle_v16f32_zz_12_zz_13_zz_16_zz_17_zz_1a_zz_1b_zz_1e_zz_1f: +; ALL: # BB#0: +; ALL-NEXT: vpxord %zmm0, %zmm0, %zmm0 +; ALL-NEXT: vunpckhps {{.*#+}} zmm0 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] +; ALL-NEXT: retq + %shuffle = shufflevector <16 x float> zeroinitializer, <16 x float> %b, <16 x i32> + ret <16 x float> %shuffle +} + define <16 x i32> @shuffle_v16i32_02_12_03_13_06_16_07_17_0a_1a_0b_1b_0e_1e_0f_1f(<16 x i32> %a, <16 x i32> %b) { ; ALL-LABEL: shuffle_v16i32_02_12_03_13_06_16_07_17_0a_1a_0b_1b_0e_1e_0f_1f: ; ALL: # BB#0: @@ -38,3 +78,97 @@ define <16 x i32> @shuffle_v16i32_02_12_03_13_06_16_07_17_0a_1a_0b_1b_0e_1e_0f_1 %shuffle = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> ret <16 x i32> %shuffle } + +define <16 x i32> @shuffle_v16i32_02_zz_03_zz_06_zz_07_zz_0a_zz_0b_zz_0e_zz_0f_zz(<16 x i32> %a, <16 x i32> %b) { +; ALL-LABEL: shuffle_v16i32_02_zz_03_zz_06_zz_07_zz_0a_zz_0b_zz_0e_zz_0f_zz: +; ALL: # BB#0: +; ALL-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; ALL-NEXT: vpunpckhdq {{.*#+}} zmm0 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] +; ALL-NEXT: retq + %shuffle = shufflevector <16 x i32> %a, <16 x i32> zeroinitializer, <16 x i32> + ret <16 x i32> %shuffle +} + +define <16 x float> @shuffle_v16f32_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_01(<16 x float> %a) { +; ALL-LABEL: shuffle_v16f32_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_01: +; ALL: # BB#0: +; ALL-NEXT: vmovdqa32 {{.*#+}} zmm1 = <2,5,u,u,7,u,10,1,0,5,u,4,7,u,10,1> +; ALL-NEXT: vpermps %zmm0, %zmm1, %zmm0 +; ALL-NEXT: retq + %c = shufflevector <16 x float> %a, <16 x float> undef, <16 x i32> + ret <16 x float> %c +} + +define <16 x i32> @shuffle_v16i32_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_01(<16 x i32> %a) { +; ALL-LABEL: shuffle_v16i32_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_01: +; ALL: # BB#0: +; ALL-NEXT: vmovdqa32 {{.*#+}} zmm1 = <2,5,u,u,7,u,10,1,0,5,u,4,7,u,10,1> +; ALL-NEXT: vpermd %zmm0, %zmm1, %zmm0 +; ALL-NEXT: retq + %c = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32> + ret <16 x i32> %c +} + +define <16 x i32> @shuffle_v16i32_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_18(<16 x i32> %a, <16 x i32> %b) { +; ALL-LABEL: shuffle_v16i32_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_18: +; ALL: # BB#0: +; ALL-NEXT: vmovdqa32 {{.*#+}} zmm2 = [15,31,14,22,13,29,4,28,11,27,10,26,9,25,8,24] +; ALL-NEXT: vpermt2d %zmm1, %zmm2, %zmm0 +; ALL-NEXT: retq + %c = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> + ret <16 x i32> %c +} + +define <16 x float> @shuffle_v16f32_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_18(<16 x float> %a, <16 x float> %b) { +; ALL-LABEL: shuffle_v16f32_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_18: +; ALL: # BB#0: +; ALL-NEXT: vmovdqa32 {{.*#+}} zmm2 = [15,31,14,22,13,29,4,28,11,27,10,26,9,25,8,24] +; ALL-NEXT: vpermt2ps %zmm1, %zmm2, %zmm0 +; ALL-NEXT: retq + %c = shufflevector <16 x float> %a, <16 x float> %b, <16 x i32> + ret <16 x float> %c +} + +define <16 x float> @shuffle_v16f32_load_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_18(<16 x float> %a, <16 x float>* %b) { +; ALL-LABEL: shuffle_v16f32_load_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_18: +; ALL: # BB#0: +; ALL-NEXT: vmovdqa32 {{.*#+}} zmm1 = [15,31,14,22,13,29,4,28,11,27,10,26,9,25,8,24] +; ALL-NEXT: vpermt2ps (%rdi), %zmm1, %zmm0 +; ALL-NEXT: retq + %c = load <16 x float>, <16 x float>* %b + %d = shufflevector <16 x float> %a, <16 x float> %c, <16 x i32> + ret <16 x float> %d +} + +define <16 x i32> @shuffle_v16i32_load_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_18(<16 x i32> %a, <16 x i32>* %b) { +; ALL-LABEL: shuffle_v16i32_load_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_18: +; ALL: # BB#0: +; ALL-NEXT: vmovdqa32 {{.*#+}} zmm1 = [15,31,14,22,13,29,4,28,11,27,10,26,9,25,8,24] +; ALL-NEXT: vpermt2d (%rdi), %zmm1, %zmm0 +; ALL-NEXT: retq + %c = load <16 x i32>, <16 x i32>* %b + %d = shufflevector <16 x i32> %a, <16 x i32> %c, <16 x i32> + ret <16 x i32> %d +} + +define <16 x i32> @shuffle_v16i32_0_1_2_19_u_u_u_u_u_u_u_u_u_u_u_u(<16 x i32> %a, <16 x i32> %b) { +; ALL-LABEL: shuffle_v16i32_0_1_2_19_u_u_u_u_u_u_u_u_u_u_u_u: +; ALL: # BB#0: +; ALL-NEXT: vmovdqa32 {{.*#+}} zmm2 = <0,1,2,19,u,u,u,u,u,u,u,u,u,u,u,u> +; ALL-NEXT: vpermt2d %zmm1, %zmm2, %zmm0 +; ALL-NEXT: retq + %c = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> + ret <16 x i32> %c +} + +define <8 x float> @shuffle_v16f32_extract_256(float* %RET, float* %a) { +; ALL-LABEL: shuffle_v16f32_extract_256: +; ALL: # BB#0: +; ALL-NEXT: vmovups (%rsi), %zmm0 +; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 +; ALL-NEXT: retq + %ptr_a = bitcast float* %a to <16 x float>* + %v_a = load <16 x float>, <16 x float>* %ptr_a, align 4 + %v2 = shufflevector <16 x float> %v_a, <16 x float> undef, <8 x i32> + ret <8 x float> %v2 +} diff --git a/test/CodeGen/X86/vector-shuffle-512-v32.ll b/test/CodeGen/X86/vector-shuffle-512-v32.ll new file mode 100644 index 000000000000..ab809beb4b48 --- /dev/null +++ b/test/CodeGen/X86/vector-shuffle-512-v32.ll @@ -0,0 +1,44 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mcpu=x86-64 -mattr=+avx512f -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW + +target triple = "x86_64-unknown-unknown" + +define <32 x i16> @shuffle_v32i16_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_01_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_1f(<32 x i16> %a) { +; ALL-LABEL: shuffle_v32i16_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_01_02_05_u_u_07_u_0a_01_00_05_u_04_07_u_0a_1f: +; ALL: # BB#0: +; ALL-NEXT: vmovdqu16 {{.*#+}} zmm1 = <2,5,u,u,7,u,10,1,0,5,u,4,7,u,10,1,2,5,u,u,7,u,10,1,0,5,u,4,7,u,10,31> +; ALL-NEXT: vpermw %zmm0, %zmm1, %zmm0 +; ALL-NEXT: retq + %c = shufflevector <32 x i16> %a, <32 x i16> undef, <32 x i32> + ret <32 x i16> %c +} + +define <32 x i16> @shuffle_v32i16_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_18_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_38(<32 x i16> %a, <32 x i16> %b) { +; ALL-LABEL: shuffle_v32i16_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_18_0f_1f_0e_16_0d_1d_04_1e_0b_1b_0a_1a_09_19_08_38: +; ALL: # BB#0: +; ALL-NEXT: vmovdqu16 {{.*#+}} zmm2 = [15,31,14,22,13,29,4,28,11,27,10,26,9,25,8,24,15,31,14,22,13,29,4,28,11,27,10,26,9,25,8,56] +; ALL-NEXT: vpermt2w %zmm1, %zmm2, %zmm0 +; ALL-NEXT: retq + %c = shufflevector <32 x i16> %a, <32 x i16> %b, <32 x i32> + ret <32 x i16> %c +} + +define <32 x i16> @shuffle_v16i32_0_32_1_33_2_34_3_35_8_40_9_41_u_u_u_u(<32 x i16> %a, <32 x i16> %b) { +; ALL-LABEL: shuffle_v16i32_0_32_1_33_2_34_3_35_8_40_9_41_u_u_u_u: +; ALL: # BB#0: +; ALL-NEXT: vmovdqu16 {{.*#+}} zmm2 = <0,32,1,33,2,34,3,35,8,40,9,41,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u> +; ALL-NEXT: vpermt2w %zmm1, %zmm2, %zmm0 +; ALL-NEXT: retq + %c = shufflevector <32 x i16> %a, <32 x i16> %b, <32 x i32> + ret <32 x i16> %c +} + +define <32 x i16> @shuffle_v16i32_4_36_5_37_6_38_7_39_12_44_13_45_u_u_u_u(<32 x i16> %a, <32 x i16> %b) { +; ALL-LABEL: shuffle_v16i32_4_36_5_37_6_38_7_39_12_44_13_45_u_u_u_u: +; ALL: # BB#0: +; ALL-NEXT: vmovdqu16 {{.*#+}} zmm2 = <4,36,5,37,6,38,7,39,12,44,13,45,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u> +; ALL-NEXT: vpermt2w %zmm1, %zmm2, %zmm0 +; ALL-NEXT: retq + %c = shufflevector <32 x i16> %a, <32 x i16> %b, <32 x i32> + ret <32 x i16> %c +} diff --git a/test/CodeGen/X86/vector-shuffle-512-v8.ll b/test/CodeGen/X86/vector-shuffle-512-v8.ll index 62d4af7809b6..631968f6afa2 100644 --- a/test/CodeGen/X86/vector-shuffle-512-v8.ll +++ b/test/CodeGen/X86/vector-shuffle-512-v8.ll @@ -1,1452 +1,2209 @@ -; RUN: llc < %s -mcpu=x86-64 -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512F -; RUN: llc < %s -mcpu=x86-64 -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mcpu=x86-64 -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=AVX512F +; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=AVX512F-32 target triple = "x86_64-unknown-unknown" define <8 x double> @shuffle_v8f64_00000000(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_00000000: -; ALL: # BB#0: -; ALL-NEXT: vbroadcastsd %xmm0, %zmm0 -; ALL-NEXT: retq +; AVX512F-LABEL: shuffle_v8f64_00000000: +; AVX512F: # BB#0: +; AVX512F-NEXT: vbroadcastsd %xmm0, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_00000000: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vbroadcastsd %xmm0, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_00000010(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_00000010: -; ALL: # BB#0: -; ALL-NEXT: vbroadcastsd %xmm0, %ymm1 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,0,1,0] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; AVX512F-LABEL: shuffle_v8f64_00000010: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,0,1,0] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_00000010: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_00000200(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_00000200: -; ALL: # BB#0: -; ALL-NEXT: vbroadcastsd %xmm0, %ymm1 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,0,0] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; AVX512F-LABEL: shuffle_v8f64_00000200: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,2,0,0] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_00000200: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_00003000(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_00003000: -; ALL: # BB#0: -; ALL-NEXT: vbroadcastsd %xmm0, %ymm1 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[3,0,0,0] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; AVX512F-LABEL: shuffle_v8f64_00003000: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,3,0,0,0] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_00003000: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_00040000(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_00040000: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vbroadcastsd %xmm1, %ymm1 -; ALL-NEXT: vbroadcastsd %xmm0, %ymm0 -; ALL-NEXT: vblendpd {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; AVX512F-LABEL: shuffle_v8f64_00040000: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,4,0,0,0,0] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_00040000: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,0,4,0,0,0,0,0,0,0,0,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_00500000(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_00500000: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vblendpd {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2,3] -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,0,1,0] -; ALL-NEXT: vbroadcastsd %xmm0, %ymm0 -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; AVX512F-LABEL: shuffle_v8f64_00500000: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,5,0,0,0,0,0] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_00500000: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_06000000(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_06000000: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vblendpd {{.*#+}} ymm1 = ymm0[0,1],ymm1[2],ymm0[3] -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,0] -; ALL-NEXT: vbroadcastsd %xmm0, %ymm0 -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; AVX512F-LABEL: shuffle_v8f64_06000000: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,6,0,0,0,0,0,0] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_06000000: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_70000000(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_70000000: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vblendpd {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3] -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[3,0,0,0] -; ALL-NEXT: vbroadcastsd %xmm0, %ymm0 -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; AVX512F-LABEL: shuffle_v8f64_70000000: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; AVX512F-NEXT: movl $7, %eax +; AVX512F-NEXT: vpinsrq $0, %rax, %xmm1, %xmm2 +; AVX512F-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1 +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_70000000: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX512F-32-NEXT: movl $7, %eax +; AVX512F-32-NEXT: vpinsrd $0, %eax, %xmm1, %xmm1 +; AVX512F-32-NEXT: vpxord %zmm2, %zmm2, %zmm2 +; AVX512F-32-NEXT: vinserti32x4 $0, %xmm1, %zmm2, %zmm1 +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_01014545(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_01014545: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vinsertf128 $1, %xmm1, %ymm1, %ymm1 -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; ALL-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 -; ALL-NEXT: retq +; AVX512F-LABEL: shuffle_v8f64_01014545: +; AVX512F: # BB#0: +; AVX512F-NEXT: vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1,4,5,4,5] +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_01014545: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1,4,5,4,5] +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_00112233(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_00112233: -; ALL: # BB#0: -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,0,1,1] -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,2,3,3] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; AVX512F-LABEL: shuffle_v8f64_00112233: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,1,1,2,2,3,3] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_00112233: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,1,0,1,0,2,0,2,0,3,0,3,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_00001111(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_00001111: -; ALL: # BB#0: -; ALL-NEXT: vbroadcastsd %xmm0, %ymm1 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,1,1,1] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; AVX512F-LABEL: shuffle_v8f64_00001111: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,1,1,1,1] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_00001111: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,0,0,0,1,0,1,0,1,0,1,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_81a3c5e7(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_81a3c5e7: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm2 -; ALL-NEXT: vextractf64x4 $1, %zmm1, %ymm3 -; ALL-NEXT: vblendpd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3] -; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3] -; ALL-NEXT: vinsertf64x4 $1, %ymm2, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_81a3c5e7: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,9,2,11,4,13,6,15] +; AVX512F-NEXT: vpermt2pd %zmm0, %zmm2, %zmm1 +; AVX512F-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_81a3c5e7: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,9,0,2,0,11,0,4,0,13,0,6,0,15,0] +; AVX512F-32-NEXT: vpermt2pd %zmm0, %zmm2, %zmm1 +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_08080808(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_08080808: -; ALL: # BB#0: -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; ALL-NEXT: vbroadcastsd %xmm1, %ymm1 -; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_08080808: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,8,0,8,0,8,0,8] +; AVX512F-NEXT: vpermt2pd %zmm1, %zmm2, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_08080808: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,8,0,0,0,8,0,0,0,8,0,0,0,8,0] +; AVX512F-32-NEXT: vpermt2pd %zmm1, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_08084c4c(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_08084c4c: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm2 -; ALL-NEXT: vinsertf128 $1, %xmm2, %ymm2, %ymm2 -; ALL-NEXT: vextractf64x4 $1, %zmm1, %ymm3 -; ALL-NEXT: vbroadcastsd %xmm3, %ymm3 -; ALL-NEXT: vblendpd {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2],ymm3[3] -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; ALL-NEXT: vbroadcastsd %xmm1, %ymm1 -; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3] -; ALL-NEXT: vinsertf64x4 $1, %ymm2, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_08084c4c: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,8,0,8,4,12,4,12] +; AVX512F-NEXT: vpermt2pd %zmm1, %zmm2, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_08084c4c: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,8,0,0,0,8,0,4,0,12,0,4,0,12,0] +; AVX512F-32-NEXT: vpermt2pd %zmm1, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_8823cc67(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_8823cc67: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm2 -; ALL-NEXT: vextractf64x4 $1, %zmm1, %ymm3 -; ALL-NEXT: vbroadcastsd %xmm3, %ymm3 -; ALL-NEXT: vblendpd {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3] -; ALL-NEXT: vbroadcastsd %xmm1, %ymm1 -; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3] -; ALL-NEXT: vinsertf64x4 $1, %ymm2, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_8823cc67: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,10,11,4,4,14,15] +; AVX512F-NEXT: vpermt2pd %zmm0, %zmm2, %zmm1 +; AVX512F-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_8823cc67: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,0,0,10,0,11,0,4,0,4,0,14,0,15,0] +; AVX512F-32-NEXT: vpermt2pd %zmm0, %zmm2, %zmm1 +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_9832dc76(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_9832dc76: -; ALL: # BB#0: -; ALL-NEXT: vblendpd {{.*#+}} ymm2 = ymm1[0,1],ymm0[2,3] -; ALL-NEXT: vpermilpd {{.*#+}} ymm2 = ymm2[1,0,3,2] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vextractf64x4 $1, %zmm1, %ymm1 -; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3] -; ALL-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm2, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_9832dc76: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [1,0,11,10,5,4,15,14] +; AVX512F-NEXT: vpermt2pd %zmm0, %zmm2, %zmm1 +; AVX512F-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_9832dc76: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [1,0,0,0,11,0,10,0,5,0,4,0,15,0,14,0] +; AVX512F-32-NEXT: vpermt2pd %zmm0, %zmm2, %zmm1 +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_9810dc54(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_9810dc54: -; ALL: # BB#0: -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm2 -; ALL-NEXT: vpermilpd {{.*#+}} ymm2 = ymm2[1,0,3,2] -; ALL-NEXT: vextractf64x4 $1, %zmm1, %ymm1 -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 -; ALL-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm2, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_9810dc54: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [1,0,9,8,5,4,13,12] +; AVX512F-NEXT: vpermt2pd %zmm0, %zmm2, %zmm1 +; AVX512F-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_9810dc54: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [1,0,0,0,9,0,8,0,5,0,4,0,13,0,12,0] +; AVX512F-32-NEXT: vpermt2pd %zmm0, %zmm2, %zmm1 +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_08194c5d(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_08194c5d: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm1, %ymm2 -; ALL-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,0,2,1] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm3 -; ALL-NEXT: vpermpd {{.*#+}} ymm3 = ymm3[0,1,1,3] -; ALL-NEXT: vblendpd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3] -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,0,2,1] -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,3] -; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3] -; ALL-NEXT: vinsertf64x4 $1, %ymm2, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_08194c5d: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,8,1,9,4,12,5,13] +; AVX512F-NEXT: vpermt2pd %zmm1, %zmm2, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_08194c5d: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,8,0,1,0,9,0,4,0,12,0,5,0,13,0] +; AVX512F-32-NEXT: vpermt2pd %zmm1, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_2a3b6e7f(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_2a3b6e7f: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm1, %ymm2 -; ALL-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,2,3] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm3 -; ALL-NEXT: vpermpd {{.*#+}} ymm3 = ymm3[2,1,3,3] -; ALL-NEXT: vblendpd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3] -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,2,2,3] -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,1,3,3] -; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3] -; ALL-NEXT: vinsertf64x4 $1, %ymm2, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_2a3b6e7f: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [2,10,3,11,6,14,7,15] +; AVX512F-NEXT: vpermt2pd %zmm1, %zmm2, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_2a3b6e7f: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [2,0,10,0,3,0,11,0,6,0,14,0,7,0,15,0] +; AVX512F-32-NEXT: vpermt2pd %zmm1, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_08192a3b(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_08192a3b: -; ALL: # BB#0: -; ALL-NEXT: vpermpd {{.*#+}} ymm2 = ymm1[0,2,2,3] -; ALL-NEXT: vpermpd {{.*#+}} ymm3 = ymm0[2,1,3,3] -; ALL-NEXT: vblendpd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3] -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,0,2,1] -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,3] -; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3] -; ALL-NEXT: vinsertf64x4 $1, %ymm2, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_08192a3b: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,8,1,9,2,10,3,11] +; AVX512F-NEXT: vpermt2pd %zmm1, %zmm2, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_08192a3b: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,8,0,1,0,9,0,2,0,10,0,3,0,11,0] +; AVX512F-32-NEXT: vpermt2pd %zmm1, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_08991abb(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_08991abb: -; ALL: # BB#0: -; ALL-NEXT: vpermpd {{.*#+}} ymm2 = ymm1[0,0,1,1] -; ALL-NEXT: vblendpd {{.*#+}} ymm2 = ymm0[0],ymm2[1,2,3] -; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3] -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,2,3,3] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm2, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_08991abb: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [8,0,1,1,9,2,3,3] +; AVX512F-NEXT: vpermt2pd %zmm0, %zmm2, %zmm1 +; AVX512F-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_08991abb: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [8,0,0,0,1,0,1,0,9,0,2,0,3,0,3,0] +; AVX512F-32-NEXT: vpermt2pd %zmm0, %zmm2, %zmm1 +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_091b2d3f(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_091b2d3f: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm1, %ymm2 -; ALL-NEXT: vpermpd {{.*#+}} ymm3 = ymm0[2,1,3,3] -; ALL-NEXT: vblendpd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3] -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,3] -; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3] -; ALL-NEXT: vinsertf64x4 $1, %ymm2, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_091b2d3f: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,9,1,11,2,13,3,15] +; AVX512F-NEXT: vpermt2pd %zmm1, %zmm2, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_091b2d3f: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,9,0,1,0,11,0,2,0,13,0,3,0,15,0] +; AVX512F-32-NEXT: vpermt2pd %zmm1, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_09ab1def(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_09ab1def: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm1, %ymm2 -; ALL-NEXT: vpermilpd {{.*#+}} ymm3 = ymm0[1,0,2,2] -; ALL-NEXT: vblendpd {{.*#+}} ymm2 = ymm3[0],ymm2[1,2,3] -; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3] -; ALL-NEXT: vinsertf64x4 $1, %ymm2, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_09ab1def: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [8,1,2,3,9,5,6,7] +; AVX512F-NEXT: vpermt2pd %zmm0, %zmm2, %zmm1 +; AVX512F-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_09ab1def: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [8,0,1,0,2,0,3,0,9,0,5,0,6,0,7,0] +; AVX512F-32-NEXT: vpermt2pd %zmm0, %zmm2, %zmm1 +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_00014445(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_00014445: -; ALL: # BB#0: -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,0,0,1] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,0,0,1] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_00014445: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,1,4,4,4,5] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_00014445: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,0,1,0,4,0,4,0,4,0,5,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_00204464(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_00204464: -; ALL: # BB#0: -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,0,2,0] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,0,2,0] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_00204464: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,2,0,4,4,6,4] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_00204464: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,2,0,0,0,4,0,4,0,6,0,4,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_03004744(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_03004744: -; ALL: # BB#0: -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,3,0,0] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,3,0,0] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_03004744: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,3,0,0,4,7,4,4] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_03004744: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,3,0,0,0,0,0,4,0,7,0,4,0,4,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_10005444(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_10005444: -; ALL: # BB#0: -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[1,0,0,0] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,0,0] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_10005444: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,0,0,5,4,4,4] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_10005444: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,0,0,0,0,0,0,5,0,4,0,4,0,4,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_22006644(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_22006644: -; ALL: # BB#0: -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[2,2,0,0] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,2,0,0] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_22006644: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [2,2,0,0,6,6,4,4] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_22006644: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [2,0,2,0,0,0,0,0,6,0,6,0,4,0,4,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_33307774(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_33307774: -; ALL: # BB#0: -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[3,3,3,0] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[3,3,3,0] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_33307774: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [3,3,3,0,7,7,7,4] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_33307774: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [3,0,3,0,3,0,0,0,7,0,7,0,7,0,4,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_32107654(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_32107654: -; ALL: # BB#0: -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[3,2,1,0] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[3,2,1,0] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_32107654: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [3,2,1,0,7,6,5,4] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_32107654: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [3,0,2,0,1,0,0,0,7,0,6,0,5,0,4,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_00234467(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_00234467: -; ALL: # BB#0: -; ALL-NEXT: vpermilpd {{.*#+}} ymm1 = ymm0[0,0,2,3] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,3] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_00234467: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,2,3,4,4,6,7] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_00234467: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,2,0,3,0,4,0,4,0,6,0,7,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_00224466(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_00224466: -; ALL: # BB#0: -; ALL-NEXT: vmovddup {{.*#+}} ymm1 = ymm0[0,0,2,2] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_00224466: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,2,2,4,4,6,6] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_00224466: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,2,0,2,0,4,0,4,0,6,0,6,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_10325476(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_10325476: -; ALL: # BB#0: -; ALL-NEXT: vpermilpd {{.*#+}} ymm1 = ymm0[1,0,3,2] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_10325476: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,3,2,5,4,7,6] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_10325476: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,0,0,3,0,2,0,5,0,4,0,7,0,6,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_11335577(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_11335577: -; ALL: # BB#0: -; ALL-NEXT: vpermilpd {{.*#+}} ymm1 = ymm0[1,1,3,3] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,1,3,3] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_11335577: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,1,3,3,5,5,7,7] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_11335577: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,1,0,3,0,3,0,5,0,5,0,7,0,7,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_10235467(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_10235467: -; ALL: # BB#0: -; ALL-NEXT: vpermilpd {{.*#+}} ymm1 = ymm0[1,0,2,3] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,0,2,3] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_10235467: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,2,3,5,4,6,7] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_10235467: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,0,0,2,0,3,0,5,0,4,0,6,0,7,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_10225466(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_10225466: -; ALL: # BB#0: -; ALL-NEXT: vpermilpd {{.*#+}} ymm1 = ymm0[1,0,2,2] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[1,0,2,2] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_10225466: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,2,2,5,4,6,6] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_10225466: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,0,0,2,0,2,0,5,0,4,0,6,0,6,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_00015444(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_00015444: -; ALL: # BB#0: -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,0,0,1] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,0,0] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_00015444: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,1,5,4,4,4] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_00015444: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,0,1,0,5,0,4,0,4,0,4,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_00204644(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_00204644: -; ALL: # BB#0: -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,0,2,0] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,0,0] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_00204644: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,2,0,4,6,4,4] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_00204644: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,2,0,0,0,4,0,6,0,4,0,4,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_03004474(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_03004474: -; ALL: # BB#0: -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,3,0,0] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,0,3,0] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_03004474: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,3,0,0,4,4,7,4] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_03004474: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,3,0,0,0,0,0,4,0,4,0,7,0,4,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_10004444(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_10004444: -; ALL: # BB#0: -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[1,0,0,0] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vbroadcastsd %xmm0, %ymm0 -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_10004444: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,0,0,4,4,4,4] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_10004444: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,0,0,0,0,0,0,4,0,4,0,4,0,4,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_22006446(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_22006446: -; ALL: # BB#0: -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[2,2,0,0] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,0,0,2] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_22006446: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [2,2,0,0,6,4,4,6] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_22006446: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [2,0,2,0,0,0,0,0,6,0,4,0,4,0,6,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_33307474(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_33307474: -; ALL: # BB#0: -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[3,3,3,0] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[3,0,3,0] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_33307474: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [3,3,3,0,7,4,7,4] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_33307474: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [3,0,3,0,3,0,0,0,7,0,4,0,7,0,4,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_32104567(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_32104567: -; ALL: # BB#0: -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[3,2,1,0] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_32104567: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [3,2,1,0,4,5,6,7] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_32104567: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [3,0,2,0,1,0,0,0,4,0,5,0,6,0,7,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_00236744(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_00236744: -; ALL: # BB#0: -; ALL-NEXT: vpermilpd {{.*#+}} ymm1 = ymm0[0,0,2,3] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,3,0,0] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_00236744: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,2,3,6,7,4,4] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_00236744: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,2,0,3,0,6,0,7,0,4,0,4,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_00226644(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_00226644: -; ALL: # BB#0: -; ALL-NEXT: vmovddup {{.*#+}} ymm1 = ymm0[0,0,2,2] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,2,0,0] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_00226644: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,2,2,6,6,4,4] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_00226644: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,2,0,2,0,6,0,6,0,4,0,4,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_10324567(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_10324567: -; ALL: # BB#0: -; ALL-NEXT: vpermilpd {{.*#+}} ymm1 = ymm0[1,0,3,2] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_10324567: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,3,2,4,5,6,7] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_10324567: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,0,0,3,0,2,0,4,0,5,0,6,0,7,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_11334567(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_11334567: -; ALL: # BB#0: -; ALL-NEXT: vpermilpd {{.*#+}} ymm1 = ymm0[1,1,3,3] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_11334567: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,1,3,3,4,5,6,7] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_11334567: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,1,0,3,0,3,0,4,0,5,0,6,0,7,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_01235467(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_01235467: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vpermilpd {{.*#+}} ymm1 = ymm1[1,0,2,3] -; ALL-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_01235467: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,5,4,6,7] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_01235467: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,1,0,2,0,3,0,5,0,4,0,6,0,7,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_01235466(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_01235466: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vpermilpd {{.*#+}} ymm1 = ymm1[1,0,2,2] -; ALL-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_01235466: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,5,4,6,6] +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_01235466: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,1,0,2,0,3,0,5,0,4,0,6,0,6,0] +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_002u6u44(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_002u6u44: -; ALL: # BB#0: -; ALL-NEXT: vmovddup {{.*#+}} ymm1 = ymm0[0,0,2,2] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,1,0,0] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_002u6u44: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = <0,0,2,u,6,u,4,4> +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_002u6u44: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = <0,0,0,0,2,0,u,u,6,0,u,u,4,0,4,0> +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_00uu66uu(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_00uu66uu: -; ALL: # BB#0: -; ALL-NEXT: vbroadcastsd %xmm0, %ymm1 -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,2,2,3] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_00uu66uu: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = <0,0,u,u,6,6,u,u> +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_00uu66uu: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = <0,0,0,0,u,u,u,u,6,0,6,0,u,u,u,u> +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_103245uu(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_103245uu: -; ALL: # BB#0: -; ALL-NEXT: vpermilpd {{.*#+}} ymm1 = ymm0[1,0,3,2] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_103245uu: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = <1,0,3,2,4,5,u,u> +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_103245uu: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = <1,0,0,0,3,0,2,0,4,0,5,0,u,u,u,u> +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_1133uu67(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_1133uu67: -; ALL: # BB#0: -; ALL-NEXT: vpermilpd {{.*#+}} ymm1 = ymm0[1,1,3,3] -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_1133uu67: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = <1,1,3,3,u,u,6,7> +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_1133uu67: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = <1,0,1,0,3,0,3,0,u,u,u,u,6,0,7,0> +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_0uu354uu(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_0uu354uu: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vpermilpd {{.*#+}} ymm1 = ymm1[1,0,2,2] -; ALL-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_0uu354uu: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = <0,u,u,3,5,4,u,u> +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_0uu354uu: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = <0,0,u,u,u,u,3,0,5,0,4,0,u,u,u,u> +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_uuu3uu66(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_uuu3uu66: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vmovddup {{.*#+}} ymm1 = ymm1[0,0,2,2] -; ALL-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_uuu3uu66: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = +; AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_uuu3uu66: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = +; AVX512F-32-NEXT: vpermpd %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_c348cda0(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_c348cda0: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm2 -; ALL-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3],ymm2[0,1] -; ALL-NEXT: vextractf64x4 $1, %zmm1, %ymm3 -; ALL-NEXT: vbroadcastsd %xmm1, %ymm4 -; ALL-NEXT: vblendpd {{.*#+}} ymm4 = ymm3[0,1,2],ymm4[3] -; ALL-NEXT: vblendpd {{.*#+}} ymm2 = ymm4[0],ymm2[1,2],ymm4[3] -; ALL-NEXT: vblendpd {{.*#+}} ymm1 = ymm3[0,1],ymm1[2],ymm3[3] -; ALL-NEXT: vbroadcastsd %xmm0, %ymm0 -; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3] -; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm2, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_c348cda0: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [4,11,12,0,4,5,2,8] +; AVX512F-NEXT: vpermt2pd %zmm0, %zmm2, %zmm1 +; AVX512F-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_c348cda0: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [4,0,11,0,12,0,0,0,4,0,5,0,2,0,8,0] +; AVX512F-32-NEXT: vpermt2pd %zmm0, %zmm2, %zmm1 +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x double> @shuffle_v8f64_f511235a(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_f511235a: -; ALL: # BB#0: -; ALL-NEXT: vextractf64x4 $1, %zmm0, %ymm2 -; ALL-NEXT: vblendpd {{.*#+}} ymm3 = ymm0[0],ymm2[1],ymm0[2,3] -; ALL-NEXT: vpermpd {{.*#+}} ymm3 = ymm3[2,3,1,3] -; ALL-NEXT: vmovddup {{.*#+}} ymm4 = ymm1[0,0,2,2] -; ALL-NEXT: vblendpd {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3] -; ALL-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,1] -; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3] -; ALL-NEXT: vextractf64x4 $1, %zmm1, %ymm1 -; ALL-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[3,1,2,3] -; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3] -; ALL-NEXT: vinsertf64x4 $1, %ymm3, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_f511235a: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [15,5,1,1,2,3,5,10] +; AVX512F-NEXT: vpermt2pd %zmm1, %zmm2, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_f511235a: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [15,0,5,0,1,0,1,0,2,0,3,0,5,0,10,0] +; AVX512F-32-NEXT: vpermt2pd %zmm1, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } define <8 x i64> @shuffle_v8i64_00000000(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_00000000: -; ALL: # BB#0: -; ALL-NEXT: vpbroadcastq %xmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_00000000: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpbroadcastq %xmm0, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_00000000: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpbroadcastq %xmm0, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_00000010(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_00000010: -; ALL: # BB#0: -; ALL-NEXT: vpbroadcastq %xmm0, %ymm1 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_00000010: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,0,1,0] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_00000010: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_00000200(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_00000200: -; ALL: # BB#0: -; ALL-NEXT: vpbroadcastq %xmm0, %ymm1 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,0,0] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_00000200: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,2,0,0] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_00000200: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_00003000(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_00003000: -; ALL: # BB#0: -; ALL-NEXT: vpbroadcastq %xmm0, %ymm1 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,0,0,0] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_00003000: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,3,0,0,0] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_00003000: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_00040000(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_00040000: -; ALL: # BB#0: -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vpbroadcastq %xmm1, %ymm1 -; ALL-NEXT: vpbroadcastq %xmm0, %ymm0 -; ALL-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm1[6,7] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_00040000: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,4,0,0,0,0] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_00040000: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,0,4,0,0,0,0,0,0,0,0,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_00500000(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_00500000: -; ALL: # BB#0: -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7] -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,1,0] -; ALL-NEXT: vpbroadcastq %xmm0, %ymm0 -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_00500000: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,5,0,0,0,0,0] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_00500000: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_06000000(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_06000000: -; ALL: # BB#0: -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7] -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,0,0] -; ALL-NEXT: vpbroadcastq %xmm0, %ymm0 -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_06000000: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,6,0,0,0,0,0,0] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_06000000: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_70000000(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_70000000: -; ALL: # BB#0: -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm1[6,7] -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm1[3,0,0,0] -; ALL-NEXT: vpbroadcastq %xmm0, %ymm0 -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_70000000: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; AVX512F-NEXT: movl $7, %eax +; AVX512F-NEXT: vpinsrq $0, %rax, %xmm1, %xmm2 +; AVX512F-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1 +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_70000000: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX512F-32-NEXT: movl $7, %eax +; AVX512F-32-NEXT: vpinsrd $0, %eax, %xmm1, %xmm1 +; AVX512F-32-NEXT: vpxord %zmm2, %zmm2, %zmm2 +; AVX512F-32-NEXT: vinserti32x4 $0, %xmm1, %zmm2, %zmm1 +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_01014545(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_01014545: -; ALL: # BB#0: -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vinserti128 $1, %xmm1, %ymm1, %ymm1 -; ALL-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 -; ALL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; ALL-NEXT: retq +; AVX512F-LABEL: shuffle_v8i64_01014545: +; AVX512F: # BB#0: +; AVX512F-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1,4,5,4,5] +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_01014545: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1,4,5,4,5] +; AVX512F-32-NEXT: retl + %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_00112233(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_00112233: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,0,1,1] -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,3,3] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_00112233: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,1,1,2,2,3,3] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_00112233: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,1,0,1,0,2,0,2,0,3,0,3,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_00001111(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_00001111: -; ALL: # BB#0: -; ALL-NEXT: vpbroadcastq %xmm0, %ymm1 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,1,1,1] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_00001111: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,1,1,1,1] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_00001111: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,0,0,0,1,0,1,0,1,0,1,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_81a3c5e7(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_81a3c5e7: -; ALL: # BB#0: -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm2 -; ALL-NEXT: vextracti64x4 $1, %zmm1, %ymm3 -; ALL-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7] -; ALL-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7] -; ALL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_81a3c5e7: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,9,2,11,4,13,6,15] +; AVX512F-NEXT: vpermt2q %zmm0, %zmm2, %zmm1 +; AVX512F-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_81a3c5e7: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,9,0,2,0,11,0,4,0,13,0,6,0,15,0] +; AVX512F-32-NEXT: vpermt2q %zmm0, %zmm2, %zmm1 +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_08080808(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_08080808: -; ALL: # BB#0: -; ALL-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 -; ALL-NEXT: vpbroadcastq %xmm1, %ymm1 -; ALL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_08080808: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,8,0,8,0,8,0,8] +; AVX512F-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_08080808: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,8,0,0,0,8,0,0,0,8,0,0,0,8,0] +; AVX512F-32-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_08084c4c(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_08084c4c: -; ALL: # BB#0: -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm2 -; ALL-NEXT: vinserti128 $1, %xmm2, %ymm2, %ymm2 -; ALL-NEXT: vextracti64x4 $1, %zmm1, %ymm3 -; ALL-NEXT: vpbroadcastq %xmm3, %ymm3 -; ALL-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7] -; ALL-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0 -; ALL-NEXT: vpbroadcastq %xmm1, %ymm1 -; ALL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7] -; ALL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_08084c4c: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,8,0,8,4,12,4,12] +; AVX512F-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_08084c4c: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,8,0,0,0,8,0,4,0,12,0,4,0,12,0] +; AVX512F-32-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_8823cc67(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_8823cc67: -; ALL: # BB#0: -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm2 -; ALL-NEXT: vextracti64x4 $1, %zmm1, %ymm3 -; ALL-NEXT: vpbroadcastq %xmm3, %ymm3 -; ALL-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7] -; ALL-NEXT: vpbroadcastq %xmm1, %ymm1 -; ALL-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] -; ALL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_8823cc67: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,10,11,4,4,14,15] +; AVX512F-NEXT: vpermt2q %zmm0, %zmm2, %zmm1 +; AVX512F-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_8823cc67: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,0,0,10,0,11,0,4,0,4,0,14,0,15,0] +; AVX512F-32-NEXT: vpermt2q %zmm0, %zmm2, %zmm1 +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_9832dc76(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_9832dc76: -; ALL: # BB#0: -; ALL-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7] -; ALL-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[2,3,0,1,6,7,4,5] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vextracti64x4 $1, %zmm1, %ymm1 -; ALL-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] -; ALL-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,3,0,1,6,7,4,5] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_9832dc76: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [1,0,11,10,5,4,15,14] +; AVX512F-NEXT: vpermt2q %zmm0, %zmm2, %zmm1 +; AVX512F-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_9832dc76: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [1,0,0,0,11,0,10,0,5,0,4,0,15,0,14,0] +; AVX512F-32-NEXT: vpermt2q %zmm0, %zmm2, %zmm1 +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_9810dc54(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_9810dc54: -; ALL: # BB#0: -; ALL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm2 -; ALL-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[2,3,0,1,6,7,4,5] -; ALL-NEXT: vextracti64x4 $1, %zmm1, %ymm1 -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 -; ALL-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,3,0,1,6,7,4,5] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_9810dc54: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [1,0,9,8,5,4,13,12] +; AVX512F-NEXT: vpermt2q %zmm0, %zmm2, %zmm1 +; AVX512F-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_9810dc54: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [1,0,0,0,9,0,8,0,5,0,4,0,13,0,12,0] +; AVX512F-32-NEXT: vpermt2q %zmm0, %zmm2, %zmm1 +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_08194c5d(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_08194c5d: -; ALL: # BB#0: -; ALL-NEXT: vextracti64x4 $1, %zmm1, %ymm2 -; ALL-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm3 -; ALL-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,1,3] -; ALL-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7] -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1] -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3] -; ALL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7] -; ALL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_08194c5d: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,8,1,9,4,12,5,13] +; AVX512F-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_08194c5d: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,8,0,1,0,9,0,4,0,12,0,5,0,13,0] +; AVX512F-32-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_2a3b6e7f(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_2a3b6e7f: -; ALL: # BB#0: -; ALL-NEXT: vextracti64x4 $1, %zmm1, %ymm2 -; ALL-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm3 -; ALL-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,1,3,3] -; ALL-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7] -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3] -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3] -; ALL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7] -; ALL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_2a3b6e7f: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [2,10,3,11,6,14,7,15] +; AVX512F-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_2a3b6e7f: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [2,0,10,0,3,0,11,0,6,0,14,0,7,0,15,0] +; AVX512F-32-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_08192a3b(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_08192a3b: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm2 = ymm1[0,2,2,3] -; ALL-NEXT: vpermq {{.*#+}} ymm3 = ymm0[2,1,3,3] -; ALL-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7] -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1] -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3] -; ALL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7] -; ALL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_08192a3b: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,8,1,9,2,10,3,11] +; AVX512F-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_08192a3b: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,8,0,1,0,9,0,2,0,10,0,3,0,11,0] +; AVX512F-32-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_08991abb(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_08991abb: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm2 = ymm1[0,0,1,1] -; ALL-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0,1],ymm2[2,3,4,5,6,7] -; ALL-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5,6,7] -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,2,3,3] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_08991abb: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [8,0,1,1,9,2,3,3] +; AVX512F-NEXT: vpermt2q %zmm0, %zmm2, %zmm1 +; AVX512F-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_08991abb: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [8,0,0,0,1,0,1,0,9,0,2,0,3,0,3,0] +; AVX512F-32-NEXT: vpermt2q %zmm0, %zmm2, %zmm1 +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_091b2d3f(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_091b2d3f: -; ALL: # BB#0: -; ALL-NEXT: vextracti64x4 $1, %zmm1, %ymm2 -; ALL-NEXT: vpermq {{.*#+}} ymm3 = ymm0[2,1,3,3] -; ALL-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7] -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3] -; ALL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7] -; ALL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_091b2d3f: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,9,1,11,2,13,3,15] +; AVX512F-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_091b2d3f: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,9,0,1,0,11,0,2,0,13,0,3,0,15,0] +; AVX512F-32-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_09ab1def(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_09ab1def: -; ALL: # BB#0: -; ALL-NEXT: vextracti64x4 $1, %zmm1, %ymm2 -; ALL-NEXT: vpshufd {{.*#+}} ymm3 = ymm0[2,3,2,3,6,7,6,7] -; ALL-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5,6,7] -; ALL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3,4,5,6,7] -; ALL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_09ab1def: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [8,1,2,3,9,5,6,7] +; AVX512F-NEXT: vpermt2q %zmm0, %zmm2, %zmm1 +; AVX512F-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_09ab1def: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [8,0,1,0,2,0,3,0,9,0,5,0,6,0,7,0] +; AVX512F-32-NEXT: vpermt2q %zmm0, %zmm2, %zmm1 +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_00014445(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_00014445: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,0,0,1] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,0,1] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_00014445: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,1,4,4,4,5] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_00014445: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,0,1,0,4,0,4,0,4,0,5,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_00204464(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_00204464: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,0,2,0] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,0] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_00204464: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,2,0,4,4,6,4] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_00204464: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,2,0,0,0,4,0,4,0,6,0,4,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_03004744(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_03004744: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,3,0,0] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,0,0] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_03004744: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,3,0,0,4,7,4,4] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_03004744: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,3,0,0,0,0,0,4,0,7,0,4,0,4,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_10005444(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_10005444: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[1,0,0,0] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,0,0,0] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_10005444: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,0,0,5,4,4,4] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_10005444: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,0,0,0,0,0,0,5,0,4,0,4,0,4,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_22006644(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_22006644: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,2,0,0] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,0,0] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_22006644: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [2,2,0,0,6,6,4,4] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_22006644: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [2,0,2,0,0,0,0,0,6,0,6,0,4,0,4,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_33307774(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_33307774: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,3,3,0] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,3,3,0] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_33307774: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [3,3,3,0,7,7,7,4] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_33307774: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [3,0,3,0,3,0,0,0,7,0,7,0,7,0,4,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_32107654(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_32107654: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_32107654: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [3,2,1,0,7,6,5,4] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_32107654: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [3,0,2,0,1,0,0,0,7,0,6,0,5,0,4,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_00234467(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_00234467: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,0,2,3] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,3] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_00234467: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,2,3,4,4,6,7] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_00234467: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,2,0,3,0,4,0,4,0,6,0,7,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_00224466(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_00224466: -; ALL: # BB#0: -; ALL-NEXT: vpshufd {{.*#+}} ymm1 = ymm0[0,1,0,1,4,5,4,5] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[0,1,0,1,4,5,4,5] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_00224466: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,2,2,4,4,6,6] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_00224466: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,2,0,2,0,4,0,4,0,6,0,6,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_10325476(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_10325476: -; ALL: # BB#0: -; ALL-NEXT: vpshufd {{.*#+}} ymm1 = ymm0[2,3,0,1,6,7,4,5] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,3,0,1,6,7,4,5] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_10325476: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,3,2,5,4,7,6] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_10325476: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,0,0,3,0,2,0,5,0,4,0,7,0,6,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_11335577(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_11335577: -; ALL: # BB#0: -; ALL-NEXT: vpshufd {{.*#+}} ymm1 = ymm0[2,3,2,3,6,7,6,7] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,3,2,3,6,7,6,7] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_11335577: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,1,3,3,5,5,7,7] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_11335577: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,1,0,3,0,3,0,5,0,5,0,7,0,7,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_10235467(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_10235467: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[1,0,2,3] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,0,2,3] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_10235467: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,2,3,5,4,6,7] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_10235467: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,0,0,2,0,3,0,5,0,4,0,6,0,7,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_10225466(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_10225466: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[1,0,2,2] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,0,2,2] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_10225466: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,2,2,5,4,6,6] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_10225466: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,0,0,2,0,2,0,5,0,4,0,6,0,6,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_00015444(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_00015444: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,0,0,1] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,0,0,0] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_00015444: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,1,5,4,4,4] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_00015444: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,0,1,0,5,0,4,0,4,0,4,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_00204644(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_00204644: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,0,2,0] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,0,0] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_00204644: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,2,0,4,6,4,4] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_00204644: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,2,0,0,0,4,0,6,0,4,0,4,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_03004474(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_03004474: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,3,0,0] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,3,0] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_03004474: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,3,0,0,4,4,7,4] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_03004474: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,3,0,0,0,0,0,4,0,4,0,7,0,4,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_10004444(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_10004444: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[1,0,0,0] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpbroadcastq %xmm0, %ymm0 -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_10004444: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,0,0,4,4,4,4] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_10004444: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,0,0,0,0,0,0,4,0,4,0,4,0,4,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_22006446(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_22006446: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,2,0,0] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,0,0,2] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_22006446: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [2,2,0,0,6,4,4,6] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_22006446: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [2,0,2,0,0,0,0,0,6,0,4,0,4,0,6,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_33307474(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_33307474: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,3,3,0] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,0,3,0] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_33307474: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [3,3,3,0,7,4,7,4] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_33307474: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [3,0,3,0,3,0,0,0,7,0,4,0,7,0,4,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_32104567(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_32104567: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_32104567: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [3,2,1,0,4,5,6,7] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_32104567: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [3,0,2,0,1,0,0,0,4,0,5,0,6,0,7,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_00236744(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_00236744: -; ALL: # BB#0: -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,0,2,3] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,0,0] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_00236744: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,2,3,6,7,4,4] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_00236744: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,2,0,3,0,6,0,7,0,4,0,4,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_00226644(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_00226644: -; ALL: # BB#0: -; ALL-NEXT: vpshufd {{.*#+}} ymm1 = ymm0[0,1,0,1,4,5,4,5] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,0,0] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_00226644: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,2,2,6,6,4,4] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_00226644: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,2,0,2,0,6,0,6,0,4,0,4,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_10324567(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_10324567: -; ALL: # BB#0: -; ALL-NEXT: vpshufd {{.*#+}} ymm1 = ymm0[2,3,0,1,6,7,4,5] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_10324567: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,3,2,4,5,6,7] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_10324567: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,0,0,3,0,2,0,4,0,5,0,6,0,7,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_11334567(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_11334567: -; ALL: # BB#0: -; ALL-NEXT: vpshufd {{.*#+}} ymm1 = ymm0[2,3,2,3,6,7,6,7] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_11334567: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,1,3,3,4,5,6,7] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_11334567: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [1,0,1,0,3,0,3,0,4,0,5,0,6,0,7,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_01235467(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_01235467: -; ALL: # BB#0: -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm1[1,0,2,3] -; ALL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_01235467: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,5,4,6,7] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_01235467: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,1,0,2,0,3,0,5,0,4,0,6,0,7,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_01235466(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_01235466: -; ALL: # BB#0: -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vpermq {{.*#+}} ymm1 = ymm1[1,0,2,2] -; ALL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_01235466: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,5,4,6,6] +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_01235466: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,1,0,2,0,3,0,5,0,4,0,6,0,6,0] +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_002u6u44(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_002u6u44: -; ALL: # BB#0: -; ALL-NEXT: vpshufd {{.*#+}} ymm1 = ymm0[0,1,0,1,4,5,4,5] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,1,0,0] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_002u6u44: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = <0,0,2,u,6,u,4,4> +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_002u6u44: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = <0,0,0,0,2,0,u,u,6,0,u,u,4,0,4,0> +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_00uu66uu(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_00uu66uu: -; ALL: # BB#0: -; ALL-NEXT: vpbroadcastq %xmm0, %ymm1 -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3] -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_00uu66uu: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = <0,0,u,u,6,6,u,u> +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_00uu66uu: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = <0,0,0,0,u,u,u,u,6,0,6,0,u,u,u,u> +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_103245uu(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_103245uu: -; ALL: # BB#0: -; ALL-NEXT: vpshufd {{.*#+}} ymm1 = ymm0[2,3,0,1,6,7,4,5] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_103245uu: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = <1,0,3,2,4,5,u,u> +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_103245uu: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = <1,0,0,0,3,0,2,0,4,0,5,0,u,u,u,u> +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_1133uu67(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_1133uu67: -; ALL: # BB#0: -; ALL-NEXT: vpshufd {{.*#+}} ymm1 = ymm0[2,3,2,3,6,7,6,7] -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_1133uu67: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = <1,1,3,3,u,u,6,7> +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_1133uu67: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = <1,0,1,0,3,0,3,0,u,u,u,u,6,0,7,0> +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_0uu354uu(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_0uu354uu: -; ALL: # BB#0: -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[2,3,0,1,6,7,4,5] -; ALL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_0uu354uu: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = <0,u,u,3,5,4,u,u> +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_0uu354uu: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = <0,0,u,u,u,u,3,0,5,0,4,0,u,u,u,u> +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_uuu3uu66(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_uuu3uu66: -; ALL: # BB#0: -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm1 -; ALL-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,0,1,4,5,4,5] -; ALL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_uuu3uu66: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = +; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_uuu3uu66: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm1 = +; AVX512F-32-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x i64> @shuffle_v8i64_6caa87e5(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_6caa87e5: -; ALL: # BB#0: -; ALL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 -; ALL-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3,0,1] -; ALL-NEXT: vextracti64x4 $1, %zmm1, %ymm2 -; ALL-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0,1,2,3],ymm2[4,5],ymm1[6,7] -; ALL-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm0[2,3],ymm3[4,5],ymm0[6,7] -; ALL-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7] -; ALL-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,0,1,4,5,4,5] -; ALL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3,4,5,6,7] -; ALL-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0 -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_6caa87e5: +; AVX512F: # BB#0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [14,4,2,2,0,15,6,13] +; AVX512F-NEXT: vpermt2q %zmm0, %zmm2, %zmm1 +; AVX512F-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_6caa87e5: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vmovdqa64 {{.*#+}} zmm2 = [14,0,4,0,2,0,2,0,0,0,15,0,6,0,13,0] +; AVX512F-32-NEXT: vpermt2q %zmm0, %zmm2, %zmm1 +; AVX512F-32-NEXT: vmovaps %zmm1, %zmm0 +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } define <8 x double> @shuffle_v8f64_082a4c6e(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_082a4c6e: -; ALL: # BB#0: -; ALL-NEXT: vunpcklpd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_082a4c6e: +; AVX512F: # BB#0: +; AVX512F-NEXT: vunpcklpd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_082a4c6e: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vunpcklpd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } +define <8 x double> @shuffle_v8f64_0z2z4z6z(<8 x double> %a, <8 x double> %b) { +; +; AVX512F-LABEL: shuffle_v8f64_0z2z4z6z: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; AVX512F-NEXT: vunpcklpd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_0z2z4z6z: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; AVX512F-32-NEXT: vunpcklpd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] +; AVX512F-32-NEXT: retl + %shuffle = shufflevector <8 x double> %a, <8 x double> zeroinitializer, <8 x i32> + ret <8 x double> %shuffle +} + define <8 x i64> @shuffle_v8i64_082a4c6e(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_082a4c6e: -; ALL: # BB#0: -; ALL-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_082a4c6e: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_082a4c6e: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } +define <8 x i64> @shuffle_v8i64_z8zazcze(<8 x i64> %a, <8 x i64> %b) { +; +; AVX512F-LABEL: shuffle_v8i64_z8zazcze: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpxord %zmm0, %zmm0, %zmm0 +; AVX512F-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_z8zazcze: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpxord %zmm0, %zmm0, %zmm0 +; AVX512F-32-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] +; AVX512F-32-NEXT: retl + %shuffle = shufflevector <8 x i64> zeroinitializer, <8 x i64> %b, <8 x i32> + ret <8 x i64> %shuffle +} + define <8 x double> @shuffle_v8f64_193b5d7f(<8 x double> %a, <8 x double> %b) { -; ALL-LABEL: shuffle_v8f64_193b5d7f: -; ALL: # BB#0: -; ALL-NEXT: vunpckhpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8f64_193b5d7f: +; AVX512F: # BB#0: +; AVX512F-NEXT: vunpckhpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_193b5d7f: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vunpckhpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x double> %a, <8 x double> %b, <8 x i32> ret <8 x double> %shuffle } +define <8 x double> @shuffle_v8f64_z9zbzdzf(<8 x double> %a, <8 x double> %b) { +; +; AVX512F-LABEL: shuffle_v8f64_z9zbzdzf: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpxord %zmm0, %zmm0, %zmm0 +; AVX512F-NEXT: vunpckhpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8f64_z9zbzdzf: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpxord %zmm0, %zmm0, %zmm0 +; AVX512F-32-NEXT: vunpckhpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] +; AVX512F-32-NEXT: retl + %shuffle = shufflevector <8 x double> zeroinitializer, <8 x double> %b, <8 x i32> + ret <8 x double> %shuffle +} + define <8 x i64> @shuffle_v8i64_193b5d7f(<8 x i64> %a, <8 x i64> %b) { -; ALL-LABEL: shuffle_v8i64_193b5d7f: -; ALL: # BB#0: -; ALL-NEXT: vpunpckhqdq {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] -; ALL-NEXT: retq +; +; AVX512F-LABEL: shuffle_v8i64_193b5d7f: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpunpckhqdq {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_193b5d7f: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpunpckhqdq {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] +; AVX512F-32-NEXT: retl %shuffle = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> ret <8 x i64> %shuffle } + +define <8 x i64> @shuffle_v8i64_1z3z5z7z(<8 x i64> %a, <8 x i64> %b) { +; +; AVX512F-LABEL: shuffle_v8i64_1z3z5z7z: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; AVX512F-NEXT: vpunpckhqdq {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: shuffle_v8i64_1z3z5z7z: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; AVX512F-32-NEXT: vpunpckhqdq {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] +; AVX512F-32-NEXT: retl + %shuffle = shufflevector <8 x i64> %a, <8 x i64> zeroinitializer, <8 x i32> + ret <8 x i64> %shuffle +} + +define <8 x double> @test_vshuff64x2_512(<8 x double> %x, <8 x double> %x1) nounwind { +; AVX512F-LABEL: test_vshuff64x2_512: +; AVX512F: # BB#0: +; AVX512F-NEXT: vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5],zmm1[2,3,0,1] +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: test_vshuff64x2_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5],zmm1[2,3,0,1] +; AVX512F-32-NEXT: retl + %res = shufflevector <8 x double> %x, <8 x double> %x1, <8 x i32> + ret <8 x double> %res +} + +define <8 x double> @test_vshuff64x2_512_maskz(<8 x double> %x, <8 x double> %x1, <8 x i1> %mask) nounwind { +; AVX512F-LABEL: test_vshuff64x2_512_maskz: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpmovsxwq %xmm2, %zmm2 +; AVX512F-NEXT: vpsllq $63, %zmm2, %zmm2 +; AVX512F-NEXT: vptestmq %zmm2, %zmm2, %k1 +; AVX512F-NEXT: vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5],zmm1[2,3,0,1] +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: test_vshuff64x2_512_maskz: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpmovsxwq %xmm2, %zmm2 +; AVX512F-32-NEXT: vpsllvq .LCPI122_0, %zmm2, %zmm2 +; AVX512F-32-NEXT: vptestmq %zmm2, %zmm2, %k1 +; AVX512F-32-NEXT: vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5],zmm1[2,3,0,1] +; AVX512F-32-NEXT: retl + %y = shufflevector <8 x double> %x, <8 x double> %x1, <8 x i32> + %res = select <8 x i1> %mask, <8 x double> %y, <8 x double> zeroinitializer + ret <8 x double> %res +} + +define <8 x i64> @test_vshufi64x2_512_mask(<8 x i64> %x, <8 x i64> %x1, <8 x i1> %mask) nounwind { +; AVX512F-LABEL: test_vshufi64x2_512_mask: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpmovsxwq %xmm2, %zmm2 +; AVX512F-NEXT: vpsllq $63, %zmm2, %zmm2 +; AVX512F-NEXT: vptestmq %zmm2, %zmm2, %k1 +; AVX512F-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5],zmm1[2,3,0,1] +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: test_vshufi64x2_512_mask: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpmovsxwq %xmm2, %zmm2 +; AVX512F-32-NEXT: vpsllvq .LCPI123_0, %zmm2, %zmm2 +; AVX512F-32-NEXT: vptestmq %zmm2, %zmm2, %k1 +; AVX512F-32-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5],zmm1[2,3,0,1] +; AVX512F-32-NEXT: retl + %y = shufflevector <8 x i64> %x, <8 x i64> %x1, <8 x i32> + %res = select <8 x i1> %mask, <8 x i64> %y, <8 x i64> %x + ret <8 x i64> %res +} + +define <8 x double> @test_vshuff64x2_512_mem(<8 x double> %x, <8 x double> *%ptr) nounwind { +; AVX512F-LABEL: test_vshuff64x2_512_mem: +; AVX512F: # BB#0: +; AVX512F-NEXT: vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5],mem[2,3,0,1] +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: test_vshuff64x2_512_mem: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5],mem[2,3,0,1] +; AVX512F-32-NEXT: retl + %x1 = load <8 x double>,<8 x double> *%ptr,align 1 + %res = shufflevector <8 x double> %x, <8 x double> %x1, <8 x i32> + ret <8 x double> %res +} + +define <8 x double> @test_vshuff64x2_512_mem_mask(<8 x double> %x, <8 x double> *%ptr, <8 x i1> %mask) nounwind { +; AVX512F-LABEL: test_vshuff64x2_512_mem_mask: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpmovsxwq %xmm1, %zmm1 +; AVX512F-NEXT: vpsllq $63, %zmm1, %zmm1 +; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1 +; AVX512F-NEXT: vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5],mem[2,3,0,1] +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: test_vshuff64x2_512_mem_mask: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpmovsxwq %xmm1, %zmm1 +; AVX512F-32-NEXT: vpsllvq .LCPI125_0, %zmm1, %zmm1 +; AVX512F-32-NEXT: vptestmq %zmm1, %zmm1, %k1 +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5],mem[2,3,0,1] +; AVX512F-32-NEXT: retl + %x1 = load <8 x double>,<8 x double> *%ptr,align 1 + %y = shufflevector <8 x double> %x, <8 x double> %x1, <8 x i32> + %res = select <8 x i1> %mask, <8 x double> %y, <8 x double> %x + ret <8 x double> %res +} + +define <8 x double> @test_vshuff64x2_512_mem_maskz(<8 x double> %x, <8 x double> *%ptr, <8 x i1> %mask) nounwind { +; AVX512F-LABEL: test_vshuff64x2_512_mem_maskz: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpmovsxwq %xmm1, %zmm1 +; AVX512F-NEXT: vpsllq $63, %zmm1, %zmm1 +; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1 +; AVX512F-NEXT: vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5],mem[2,3,0,1] +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: test_vshuff64x2_512_mem_maskz: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vpmovsxwq %xmm1, %zmm1 +; AVX512F-32-NEXT: vpsllvq .LCPI126_0, %zmm1, %zmm1 +; AVX512F-32-NEXT: vptestmq %zmm1, %zmm1, %k1 +; AVX512F-32-NEXT: movl {{[0-9]+}}(%esp), %eax +; AVX512F-32-NEXT: vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5],mem[2,3,0,1] +; AVX512F-32-NEXT: retl + %x1 = load <8 x double>,<8 x double> *%ptr,align 1 + %y = shufflevector <8 x double> %x, <8 x double> %x1, <8 x i32> + %res = select <8 x i1> %mask, <8 x double> %y, <8 x double> zeroinitializer + ret <8 x double> %res +} + +define <16 x float> @test_vshuff32x4_512(<16 x float> %x, <16 x float> %x1) nounwind { +; AVX512F-LABEL: test_vshuff32x4_512: +; AVX512F: # BB#0: +; AVX512F-NEXT: vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[2,3,0,1] +; AVX512F-NEXT: retq +; +; AVX512F-32-LABEL: test_vshuff32x4_512: +; AVX512F-32: # BB#0: +; AVX512F-32-NEXT: vshuff64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[2,3,0,1] +; AVX512F-32-NEXT: retl + %res = shufflevector <16 x float> %x, <16 x float> %x1, <16 x i32> + ret <16 x float> %res +} diff --git a/test/CodeGen/X86/vector-shuffle-combining.ll b/test/CodeGen/X86/vector-shuffle-combining.ll index 92c59e2fca08..75ce9753525b 100644 --- a/test/CodeGen/X86/vector-shuffle-combining.ll +++ b/test/CodeGen/X86/vector-shuffle-combining.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mcpu=x86-64 -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 ; RUN: llc < %s -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3 ; RUN: llc < %s -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 diff --git a/test/CodeGen/X86/vector-shuffle-mmx.ll b/test/CodeGen/X86/vector-shuffle-mmx.ll index dbccd2694b07..37f9ea98949f 100644 --- a/test/CodeGen/X86/vector-shuffle-mmx.ll +++ b/test/CodeGen/X86/vector-shuffle-mmx.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=i686-darwin -mattr=+mmx,+sse2 | FileCheck --check-prefix=X32 %s ; RUN: llc < %s -mtriple=x86_64-darwin -mattr=+mmx,+sse2 | FileCheck --check-prefix=X64 %s diff --git a/test/CodeGen/X86/vector-shuffle-sse1.ll b/test/CodeGen/X86/vector-shuffle-sse1.ll index 66e53bbb7502..548de4ce6ea3 100644 --- a/test/CodeGen/X86/vector-shuffle-sse1.ll +++ b/test/CodeGen/X86/vector-shuffle-sse1.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mcpu=x86-64 -mattr=-sse2 | FileCheck %s --check-prefix=SSE1 target triple = "x86_64-unknown-unknown" diff --git a/test/CodeGen/X86/vector-shuffle-sse4a.ll b/test/CodeGen/X86/vector-shuffle-sse4a.ll index 26062335cc16..eec915d91bbb 100644 --- a/test/CodeGen/X86/vector-shuffle-sse4a.ll +++ b/test/CodeGen/X86/vector-shuffle-sse4a.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3,+sse4a | FileCheck %s --check-prefix=ALL --check-prefix=BTVER1 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+sse4a | FileCheck %s --check-prefix=ALL --check-prefix=BTVER2 @@ -5,6 +6,35 @@ ; EXTRQI ; +; A length of zero is equivalent to a bit length of 64. +define <2 x i64> @extrqi_len0_idx0(<2 x i64> %a) { +; ALL-LABEL: extrqi_len0_idx0: +; ALL: # BB#0: +; ALL-NEXT: extrq {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,7,u,u,u,u,u,u,u,u] +; ALL-NEXT: retq + %1 = tail call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> %a, i8 0, i8 0) + ret <2 x i64> %1 +} + +define <2 x i64> @extrqi_len8_idx16(<2 x i64> %a) { +; ALL-LABEL: extrqi_len8_idx16: +; ALL: # BB#0: +; ALL-NEXT: extrq {{.*#+}} xmm0 = xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[u,u,u,u,u,u,u,u] +; ALL-NEXT: retq + %1 = tail call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> %a, i8 8, i8 16) + ret <2 x i64> %1 +} + +; If the length + index exceeds the bottom 64 bits the result is undefined. +define <2 x i64> @extrqi_len32_idx48(<2 x i64> %a) { +; ALL-LABEL: extrqi_len32_idx48: +; ALL: # BB#0: +; ALL-NEXT: extrq {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u] +; ALL-NEXT: retq + %1 = tail call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> %a, i8 32, i8 48) + ret <2 x i64> %1 +} + define <16 x i8> @shuf_0zzzuuuuuuuuuuuu(<16 x i8> %a0) { ; BTVER1-LABEL: shuf_0zzzuuuuuuuuuuuu: ; BTVER1: # BB#0: @@ -36,6 +66,24 @@ define <16 x i8> @shuf_0zzzzzzz1zzzzzzz(<16 x i8> %a0) { ret <16 x i8> %s } +define <16 x i8> @shuf_2zzzzzzz3zzzzzzz(<16 x i8> %a0) { +; BTVER1-LABEL: shuf_2zzzzzzz3zzzzzzz: +; BTVER1: # BB#0: +; BTVER1-NEXT: movaps %xmm0, %xmm1 +; BTVER1-NEXT: extrq {{.*#+}} xmm1 = xmm1[3],zero,zero,zero,zero,zero,zero,zero,xmm1[u,u,u,u,u,u,u,u] +; BTVER1-NEXT: extrq {{.*#+}} xmm0 = xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[u,u,u,u,u,u,u,u] +; BTVER1-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; BTVER1-NEXT: retq +; +; BTVER2-LABEL: shuf_2zzzzzzz3zzzzzzz: +; BTVER2: # BB#0: +; BTVER2-NEXT: vpsrld $16, %xmm0, %xmm0 +; BTVER2-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; BTVER2-NEXT: retq + %s = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> + ret <16 x i8> %s +} + define <16 x i8> @shuf_01zzuuuuuuuuuuuu(<16 x i8> %a0) { ; BTVER1-LABEL: shuf_01zzuuuuuuuuuuuu: ; BTVER1: # BB#0: @@ -139,6 +187,35 @@ define <4 x i32> @shuf_0z1z(<4 x i32> %a0) { ; INSERTQI ; +; A length of zero is equivalent to a bit length of 64. +define <2 x i64> @insertqi_len0_idx0(<2 x i64> %a, <2 x i64> %b) { +; ALL-LABEL: insertqi_len0_idx0: +; ALL: # BB#0: +; ALL-NEXT: insertq {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5,6,7],xmm0[u,u,u,u,u,u,u,u] +; ALL-NEXT: retq + %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %a, <2 x i64> %b, i8 0, i8 0) + ret <2 x i64> %1 +} + +define <2 x i64> @insertqi_len8_idx16(<2 x i64> %a, <2 x i64> %b) { +; ALL-LABEL: insertqi_len8_idx16: +; ALL: # BB#0: +; ALL-NEXT: insertq {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3,4,5,6,7,u,u,u,u,u,u,u,u] +; ALL-NEXT: retq + %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %a, <2 x i64> %b, i8 8, i8 16) + ret <2 x i64> %1 +} + +; If the length + index exceeds the bottom 64 bits the result is undefined +define <2 x i64> @insertqi_len32_idx48(<2 x i64> %a, <2 x i64> %b) { +; ALL-LABEL: insertqi_len32_idx48: +; ALL: # BB#0: +; ALL-NEXT: insertq {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u] +; ALL-NEXT: retq + %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %a, <2 x i64> %b, i8 32, i8 48) + ret <2 x i64> %1 +} + define <16 x i8> @shuf_0_0_2_3_uuuu_uuuu_uuuu(<16 x i8> %a0, <16 x i8> %a1) { ; ALL-LABEL: shuf_0_0_2_3_uuuu_uuuu_uuuu: ; ALL: # BB#0: @@ -219,3 +296,66 @@ define <8 x i16> @shuf_089uuuuu(<8 x i16> %a0, <8 x i16> %a1) { %s = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> ret <8 x i16> %s } + +; +; Special Cases +; + +; Out of range. +define <16 x i8> @shuffle_8_18_uuuuuuuuuuuuuu(<16 x i8> %a, <16 x i8> %b) { +; BTVER1-LABEL: shuffle_8_18_uuuuuuuuuuuuuu: +; BTVER1: # BB#0: +; BTVER1-NEXT: psrld $16, %xmm1 +; BTVER1-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; BTVER1-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; BTVER1-NEXT: retq +; +; BTVER2-LABEL: shuffle_8_18_uuuuuuuuuuuuuu: +; BTVER2: # BB#0: +; BTVER2-NEXT: vpsrld $16, %xmm1, %xmm1 +; BTVER2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; BTVER2-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; BTVER2-NEXT: retq + %1 = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> + ret <16 x i8> %1 +} + +define <16 x i8> @shuffle_uu_0_5_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu(<16 x i8> %v) { +; BTVER1-LABEL: shuffle_uu_0_5_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu: +; BTVER1: # BB#0: +; BTVER1-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,0,5,5,4,4,5,5,4,4,5,5,6,6,7,7] +; BTVER1-NEXT: retq +; +; BTVER2-LABEL: shuffle_uu_0_5_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu: +; BTVER2: # BB#0: +; BTVER2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,0,5,5,4,4,5,5,4,4,5,5,6,6,7,7] +; BTVER2-NEXT: retq + %1 = shufflevector <16 x i8> %v, <16 x i8> zeroinitializer, <16 x i32> + ret <16 x i8> %1 +} + +define <16 x i8> @shuffle_uu_16_4_16_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu(<16 x i8> %v) { +; BTVER1-LABEL: shuffle_uu_16_4_16_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu: +; BTVER1: # BB#0: +; BTVER1-NEXT: pshufb {{.*#+}} xmm0 = xmm0[u],zero,xmm0[4],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u] +; BTVER1-NEXT: retq +; +; BTVER2-LABEL: shuffle_uu_16_4_16_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu: +; BTVER2: # BB#0: +; BTVER2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u],zero,xmm0[4],zero,xmm0[u,u,u,u,u,u,u,u,u,u,u,u] +; BTVER2-NEXT: retq + %1 = shufflevector <16 x i8> %v, <16 x i8> zeroinitializer, <16 x i32> + ret <16 x i8> %1 +} + +define <16 x i8> @shuffle_uu_uu_4_16_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu(<16 x i8> %v) { +; ALL-LABEL: shuffle_uu_uu_4_16_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu: +; ALL: # BB#0: +; ALL-NEXT: extrq {{.*#+}} xmm0 = xmm0[2,3,4],zero,zero,zero,zero,zero,xmm0[u,u,u,u,u,u,u,u] +; ALL-NEXT: retq + %1 = shufflevector <16 x i8> %v, <16 x i8> zeroinitializer, <16 x i32> + ret <16 x i8> %1 +} + +declare <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64>, i8, i8) nounwind +declare <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64>, <2 x i64>, i8, i8) nounwind diff --git a/test/CodeGen/X86/vector-shuffle-v1.ll b/test/CodeGen/X86/vector-shuffle-v1.ll new file mode 100644 index 000000000000..a387f894a067 --- /dev/null +++ b/test/CodeGen/X86/vector-shuffle-v1.ll @@ -0,0 +1,439 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mcpu=x86-64 -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F +; RUN: llc < %s -mcpu=x86-64 -mattr=+avx512bw -mattr=+avx512vl -mattr=+avx512dq| FileCheck %s --check-prefix=VL_BW_DQ + +target triple = "x86_64-unknown-unknown" + +define <2 x i1> @shuf2i1_1_0(<2 x i1> %a) { +; AVX512F-LABEL: shuf2i1_1_0: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; AVX512F-NEXT: retq +; +; VL_BW_DQ-LABEL: shuf2i1_1_0: +; VL_BW_DQ: # BB#0: +; VL_BW_DQ-NEXT: vpsllq $63, %xmm0, %xmm0 +; VL_BW_DQ-NEXT: vpmovq2m %xmm0, %k0 +; VL_BW_DQ-NEXT: vpmovm2q %k0, %xmm0 +; VL_BW_DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; VL_BW_DQ-NEXT: vpsllq $63, %xmm0, %xmm0 +; VL_BW_DQ-NEXT: vpmovq2m %xmm0, %k0 +; VL_BW_DQ-NEXT: vpmovm2q %k0, %xmm0 +; VL_BW_DQ-NEXT: retq + %b = shufflevector <2 x i1> %a, <2 x i1> undef, <2 x i32> + ret <2 x i1> %b +} + +define <2 x i1> @shuf2i1_1_2(<2 x i1> %a) { +; AVX512F-LABEL: shuf2i1_1_2: +; AVX512F: # BB#0: +; AVX512F-NEXT: movl $1, %eax +; AVX512F-NEXT: vmovq %rax, %xmm1 +; AVX512F-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7] +; AVX512F-NEXT: retq +; +; VL_BW_DQ-LABEL: shuf2i1_1_2: +; VL_BW_DQ: # BB#0: +; VL_BW_DQ-NEXT: vpsllq $63, %xmm0, %xmm0 +; VL_BW_DQ-NEXT: vpmovq2m %xmm0, %k0 +; VL_BW_DQ-NEXT: vpmovm2q %k0, %xmm0 +; VL_BW_DQ-NEXT: movb $1, %al +; VL_BW_DQ-NEXT: kmovb %eax, %k0 +; VL_BW_DQ-NEXT: vpmovm2q %k0, %xmm1 +; VL_BW_DQ-NEXT: vpalignr $8, %xmm0, %xmm1, %xmm0 +; VL_BW_DQ-NEXT: vpsllq $63, %xmm0, %xmm0 +; VL_BW_DQ-NEXT: vpmovq2m %xmm0, %k0 +; VL_BW_DQ-NEXT: vpmovm2q %k0, %xmm0 +; VL_BW_DQ-NEXT: retq + %b = shufflevector <2 x i1> %a, <2 x i1> , <2 x i32> + ret <2 x i1> %b +} + + +define <4 x i1> @shuf4i1_3_2_10(<4 x i1> %a) { +; AVX512F-LABEL: shuf4i1_3_2_10: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,2,1,0] +; AVX512F-NEXT: retq +; +; VL_BW_DQ-LABEL: shuf4i1_3_2_10: +; VL_BW_DQ: # BB#0: +; VL_BW_DQ-NEXT: vpslld $31, %xmm0, %xmm0 +; VL_BW_DQ-NEXT: vpmovd2m %xmm0, %k0 +; VL_BW_DQ-NEXT: vpmovm2d %k0, %xmm0 +; VL_BW_DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,2,1,0] +; VL_BW_DQ-NEXT: vpslld $31, %xmm0, %xmm0 +; VL_BW_DQ-NEXT: vpmovd2m %xmm0, %k0 +; VL_BW_DQ-NEXT: vpmovm2d %k0, %xmm0 +; VL_BW_DQ-NEXT: retq + %b = shufflevector <4 x i1> %a, <4 x i1> undef, <4 x i32> + ret <4 x i1> %b +} + +define <8 x i1> @shuf8i1_3_6_1_0_3_7_7_0(<8 x i64> %a, <8 x i64> %b, <8 x i64> %a1, <8 x i64> %b1) { +; AVX512F-LABEL: shuf8i1_3_6_1_0_3_7_7_0: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpcmpeqq %zmm2, %zmm0, %k1 +; AVX512F-NEXT: vpbroadcastq {{.*}}(%rip), %zmm0 +; AVX512F-NEXT: vmovdqu64 %zmm0, %zmm1 {%k1} {z} +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [3,6,1,0,3,7,7,0] +; AVX512F-NEXT: vpermq %zmm1, %zmm2, %zmm1 +; AVX512F-NEXT: vpsllq $63, %zmm1, %zmm1 +; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1 +; AVX512F-NEXT: vmovdqu64 %zmm0, %zmm0 {%k1} {z} +; AVX512F-NEXT: vpmovqw %zmm0, %xmm0 +; AVX512F-NEXT: retq +; +; VL_BW_DQ-LABEL: shuf8i1_3_6_1_0_3_7_7_0: +; VL_BW_DQ: # BB#0: +; VL_BW_DQ-NEXT: vpcmpeqq %zmm2, %zmm0, %k0 +; VL_BW_DQ-NEXT: vpmovm2q %k0, %zmm0 +; VL_BW_DQ-NEXT: vmovdqa64 {{.*#+}} zmm1 = [3,6,1,0,3,7,7,0] +; VL_BW_DQ-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; VL_BW_DQ-NEXT: vpsllq $63, %zmm0, %zmm0 +; VL_BW_DQ-NEXT: vpmovq2m %zmm0, %k0 +; VL_BW_DQ-NEXT: vpmovm2w %k0, %xmm0 +; VL_BW_DQ-NEXT: retq + %a2 = icmp eq <8 x i64> %a, %a1 + %b2 = icmp eq <8 x i64> %b, %b1 + %c = shufflevector <8 x i1> %a2, <8 x i1> %b2, <8 x i32> + ret <8 x i1> %c +} + +define <16 x i1> @shuf16i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0(<16 x i32> %a, <16 x i32> %b, <16 x i32> %a1, <16 x i32> %b1) { +; AVX512F-LABEL: shuf16i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpcmpeqd %zmm2, %zmm0, %k1 +; AVX512F-NEXT: vpcmpeqd %zmm3, %zmm1, %k2 +; AVX512F-NEXT: vpbroadcastd {{.*}}(%rip), %zmm0 +; AVX512F-NEXT: vmovdqu32 %zmm0, %zmm1 {%k2} {z} +; AVX512F-NEXT: vmovdqu32 %zmm0, %zmm2 {%k1} {z} +; AVX512F-NEXT: vmovdqa32 {{.*#+}} zmm3 = [3,6,22,12,3,7,7,0,3,6,1,13,3,21,7,0] +; AVX512F-NEXT: vpermt2d %zmm1, %zmm3, %zmm2 +; AVX512F-NEXT: vpslld $31, %zmm2, %zmm1 +; AVX512F-NEXT: vptestmd %zmm1, %zmm1, %k1 +; AVX512F-NEXT: vmovdqu32 %zmm0, %zmm0 {%k1} {z} +; AVX512F-NEXT: vpmovdb %zmm0, %xmm0 +; AVX512F-NEXT: retq +; +; VL_BW_DQ-LABEL: shuf16i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0: +; VL_BW_DQ: # BB#0: +; VL_BW_DQ-NEXT: vpcmpeqd %zmm2, %zmm0, %k0 +; VL_BW_DQ-NEXT: vpcmpeqd %zmm3, %zmm1, %k1 +; VL_BW_DQ-NEXT: vpmovm2d %k1, %zmm0 +; VL_BW_DQ-NEXT: vpmovm2d %k0, %zmm1 +; VL_BW_DQ-NEXT: vmovdqa32 {{.*#+}} zmm2 = [3,6,22,12,3,7,7,0,3,6,1,13,3,21,7,0] +; VL_BW_DQ-NEXT: vpermt2d %zmm0, %zmm2, %zmm1 +; VL_BW_DQ-NEXT: vpslld $31, %zmm1, %zmm0 +; VL_BW_DQ-NEXT: vpmovd2m %zmm0, %k0 +; VL_BW_DQ-NEXT: vpmovm2b %k0, %xmm0 +; VL_BW_DQ-NEXT: retq + %a2 = icmp eq <16 x i32> %a, %a1 + %b2 = icmp eq <16 x i32> %b, %b1 + %c = shufflevector <16 x i1> %a2, <16 x i1> %b2, <16 x i32> + ret <16 x i1> %c +} + +define <32 x i1> @shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0(<32 x i1> %a) { +; AVX512F-LABEL: shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0: +; AVX512F: # BB#0: +; AVX512F-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm0[2,3,0,1] +; AVX512F-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,6,u,u,u,u,u,u,u,u,u,u,5,u,u,19,22,u,28,19,23,23,16,19,22,17,29,19,u,23,16] +; AVX512F-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[3,6,u,12,3,7,7,0,3,6,1,13,3,u,7,0,u,u,22,u,u,u,u,u,u,u,u,u,u,21,u,u] +; AVX512F-NEXT: vmovdqa {{.*#+}} ymm2 = [255,255,0,255,255,255,255,255,255,255,255,255,255,0,255,255,0,0,255,0,0,0,0,0,0,0,0,0,0,255,0,0] +; AVX512F-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0 +; AVX512F-NEXT: retq +; +; VL_BW_DQ-LABEL: shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0: +; VL_BW_DQ: # BB#0: +; VL_BW_DQ-NEXT: vpsllw $7, %ymm0, %ymm0 +; VL_BW_DQ-NEXT: vpmovb2m %ymm0, %k0 +; VL_BW_DQ-NEXT: vpmovm2w %k0, %zmm0 +; VL_BW_DQ-NEXT: vmovdqu16 {{.*#+}} zmm1 = [3,6,22,12,3,7,7,0,3,6,1,13,3,21,7,0,3,6,22,12,3,7,7,0,3,6,1,13,3,21,7,0] +; VL_BW_DQ-NEXT: vpermw %zmm0, %zmm1, %zmm0 +; VL_BW_DQ-NEXT: vpsllw $15, %zmm0, %zmm0 +; VL_BW_DQ-NEXT: vpmovw2m %zmm0, %k0 +; VL_BW_DQ-NEXT: vpmovm2b %k0, %ymm0 +; VL_BW_DQ-NEXT: retq + %b = shufflevector <32 x i1> %a, <32 x i1> undef, <32 x i32> + ret <32 x i1> %b +} + +define <8 x i1> @shuf8i1_u_2_u_u_2_u_2_u(i8 %a) { +; AVX512F-LABEL: shuf8i1_u_2_u_u_2_u_2_u: +; AVX512F: # BB#0: +; AVX512F-NEXT: movzbl %dil, %eax +; AVX512F-NEXT: kmovw %eax, %k1 +; AVX512F-NEXT: vpbroadcastq {{.*}}(%rip), %zmm0 +; AVX512F-NEXT: vmovdqu64 %zmm0, %zmm1 {%k1} {z} +; AVX512F-NEXT: vpbroadcastq {{.*}}(%rip), %zmm2 +; AVX512F-NEXT: vpermq %zmm1, %zmm2, %zmm1 +; AVX512F-NEXT: vpsllq $63, %zmm1, %zmm1 +; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1 +; AVX512F-NEXT: vmovdqu64 %zmm0, %zmm0 {%k1} {z} +; AVX512F-NEXT: vpmovqw %zmm0, %xmm0 +; AVX512F-NEXT: retq +; +; VL_BW_DQ-LABEL: shuf8i1_u_2_u_u_2_u_2_u: +; VL_BW_DQ: # BB#0: +; VL_BW_DQ-NEXT: kmovb %edi, %k0 +; VL_BW_DQ-NEXT: vpmovm2q %k0, %zmm0 +; VL_BW_DQ-NEXT: vpbroadcastq {{.*}}(%rip), %zmm1 +; VL_BW_DQ-NEXT: vpermq %zmm0, %zmm1, %zmm0 +; VL_BW_DQ-NEXT: vpsllq $63, %zmm0, %zmm0 +; VL_BW_DQ-NEXT: vpmovq2m %zmm0, %k0 +; VL_BW_DQ-NEXT: vpmovm2w %k0, %xmm0 +; VL_BW_DQ-NEXT: retq + %b = bitcast i8 %a to <8 x i1> + %c = shufflevector < 8 x i1> %b, <8 x i1>undef, <8 x i32> + ret <8 x i1> %c +} + +define i8 @shuf8i1_10_2_9_u_3_u_2_u(i8 %a) { +; AVX512F-LABEL: shuf8i1_10_2_9_u_3_u_2_u: +; AVX512F: # BB#0: +; AVX512F-NEXT: movzbl %dil, %eax +; AVX512F-NEXT: kmovw %eax, %k1 +; AVX512F-NEXT: vpbroadcastq {{.*}}(%rip), %zmm0 {%k1} {z} +; AVX512F-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = <8,2,10,u,3,u,2,u> +; AVX512F-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 +; AVX512F-NEXT: vpsllq $63, %zmm0, %zmm0 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0 +; AVX512F-NEXT: kmovw %k0, %eax +; AVX512F-NEXT: retq +; +; VL_BW_DQ-LABEL: shuf8i1_10_2_9_u_3_u_2_u: +; VL_BW_DQ: # BB#0: +; VL_BW_DQ-NEXT: kmovb %edi, %k0 +; VL_BW_DQ-NEXT: vpmovm2q %k0, %zmm0 +; VL_BW_DQ-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; VL_BW_DQ-NEXT: vmovdqa64 {{.*#+}} zmm2 = <8,2,10,u,3,u,2,u> +; VL_BW_DQ-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 +; VL_BW_DQ-NEXT: vpsllq $63, %zmm0, %zmm0 +; VL_BW_DQ-NEXT: vpmovq2m %zmm0, %k0 +; VL_BW_DQ-NEXT: kmovb %k0, %eax +; VL_BW_DQ-NEXT: retq + %b = bitcast i8 %a to <8 x i1> + %c = shufflevector < 8 x i1> %b, <8 x i1> zeroinitializer, <8 x i32> + %d = bitcast <8 x i1> %c to i8 + ret i8 %d +} + +define i8 @shuf8i1_0_1_4_5_u_u_u_u(i8 %a) { +; AVX512F-LABEL: shuf8i1_0_1_4_5_u_u_u_u: +; AVX512F: # BB#0: +; AVX512F-NEXT: movzbl %dil, %eax +; AVX512F-NEXT: kmovw %eax, %k1 +; AVX512F-NEXT: vpbroadcastq {{.*}}(%rip), %zmm0 {%k1} {z} +; AVX512F-NEXT: vshufi64x2 $8, %zmm0, %zmm0, %zmm0 # zmm0 = zmm0[0,1,4,5,0,1,0,1] +; AVX512F-NEXT: vpsllq $63, %zmm0, %zmm0 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0 +; AVX512F-NEXT: kmovw %k0, %eax +; AVX512F-NEXT: retq +; +; VL_BW_DQ-LABEL: shuf8i1_0_1_4_5_u_u_u_u: +; VL_BW_DQ: # BB#0: +; VL_BW_DQ-NEXT: kmovb %edi, %k0 +; VL_BW_DQ-NEXT: vpmovm2q %k0, %zmm0 +; VL_BW_DQ-NEXT: vshufi64x2 $8, %zmm0, %zmm0, %zmm0 # zmm0 = zmm0[0,1,4,5,0,1,0,1] +; VL_BW_DQ-NEXT: vpsllq $63, %zmm0, %zmm0 +; VL_BW_DQ-NEXT: vpmovq2m %zmm0, %k0 +; VL_BW_DQ-NEXT: kmovb %k0, %eax +; VL_BW_DQ-NEXT: retq + %b = bitcast i8 %a to <8 x i1> + %c = shufflevector < 8 x i1> %b, <8 x i1> undef, <8 x i32> + %d = bitcast <8 x i1> %c to i8 + ret i8 %d +} + +define i8 @shuf8i1_9_6_1_0_3_7_7_0(i8 %a) { +; AVX512F-LABEL: shuf8i1_9_6_1_0_3_7_7_0: +; AVX512F: # BB#0: +; AVX512F-NEXT: movzbl %dil, %eax +; AVX512F-NEXT: kmovw %eax, %k1 +; AVX512F-NEXT: vpbroadcastq {{.*}}(%rip), %zmm0 {%k1} {z} +; AVX512F-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [8,6,1,0,3,7,7,0] +; AVX512F-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 +; AVX512F-NEXT: vpsllq $63, %zmm0, %zmm0 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0 +; AVX512F-NEXT: kmovw %k0, %eax +; AVX512F-NEXT: retq +; +; VL_BW_DQ-LABEL: shuf8i1_9_6_1_0_3_7_7_0: +; VL_BW_DQ: # BB#0: +; VL_BW_DQ-NEXT: kmovb %edi, %k0 +; VL_BW_DQ-NEXT: vpmovm2q %k0, %zmm0 +; VL_BW_DQ-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; VL_BW_DQ-NEXT: vmovdqa64 {{.*#+}} zmm2 = [8,6,1,0,3,7,7,0] +; VL_BW_DQ-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 +; VL_BW_DQ-NEXT: vpsllq $63, %zmm0, %zmm0 +; VL_BW_DQ-NEXT: vpmovq2m %zmm0, %k0 +; VL_BW_DQ-NEXT: kmovb %k0, %eax +; VL_BW_DQ-NEXT: retq + %b = bitcast i8 %a to <8 x i1> + %c = shufflevector <8 x i1> %b, <8 x i1> zeroinitializer, <8 x i32> + %d = bitcast <8 x i1>%c to i8 + ret i8 %d +} + +define i8 @shuf8i1_9_6_1_10_3_7_7_0(i8 %a) { +; AVX512F-LABEL: shuf8i1_9_6_1_10_3_7_7_0: +; AVX512F: # BB#0: +; AVX512F-NEXT: movzbl %dil, %eax +; AVX512F-NEXT: kmovw %eax, %k1 +; AVX512F-NEXT: vpbroadcastq {{.*}}(%rip), %zmm0 {%k1} {z} +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [9,1,2,10,4,5,6,7] +; AVX512F-NEXT: vpxord %zmm2, %zmm2, %zmm2 +; AVX512F-NEXT: vpermt2q %zmm0, %zmm1, %zmm2 +; AVX512F-NEXT: vpsllq $63, %zmm2, %zmm0 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0 +; AVX512F-NEXT: kmovw %k0, %eax +; AVX512F-NEXT: retq +; +; VL_BW_DQ-LABEL: shuf8i1_9_6_1_10_3_7_7_0: +; VL_BW_DQ: # BB#0: +; VL_BW_DQ-NEXT: kmovb %edi, %k0 +; VL_BW_DQ-NEXT: vpmovm2q %k0, %zmm0 +; VL_BW_DQ-NEXT: vmovdqa64 {{.*#+}} zmm1 = [9,1,2,10,4,5,6,7] +; VL_BW_DQ-NEXT: vpxord %zmm2, %zmm2, %zmm2 +; VL_BW_DQ-NEXT: vpermt2q %zmm0, %zmm1, %zmm2 +; VL_BW_DQ-NEXT: vpsllq $63, %zmm2, %zmm0 +; VL_BW_DQ-NEXT: vpmovq2m %zmm0, %k0 +; VL_BW_DQ-NEXT: kmovb %k0, %eax +; VL_BW_DQ-NEXT: retq + %b = bitcast i8 %a to <8 x i1> + %c = shufflevector <8 x i1> zeroinitializer, <8 x i1> %b, <8 x i32> + %d = bitcast <8 x i1>%c to i8 + ret i8 %d +} + +define i8 @shuf8i1__9_6_1_10_3_7_7_1(i8 %a) { +; AVX512F-LABEL: shuf8i1__9_6_1_10_3_7_7_1: +; AVX512F: # BB#0: +; AVX512F-NEXT: movzbl %dil, %eax +; AVX512F-NEXT: kmovw %eax, %k1 +; AVX512F-NEXT: movb $51, %al +; AVX512F-NEXT: movzbl %al, %eax +; AVX512F-NEXT: kmovw %eax, %k2 +; AVX512F-NEXT: vpbroadcastq {{.*}}(%rip), %zmm0 +; AVX512F-NEXT: vmovdqu64 %zmm0, %zmm1 {%k2} {z} +; AVX512F-NEXT: vmovdqu64 %zmm0, %zmm0 {%k1} {z} +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [9,6,1,0,3,7,7,1] +; AVX512F-NEXT: vpermt2q %zmm0, %zmm2, %zmm1 +; AVX512F-NEXT: vpsllq $63, %zmm1, %zmm0 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0 +; AVX512F-NEXT: kmovw %k0, %eax +; AVX512F-NEXT: retq +; +; VL_BW_DQ-LABEL: shuf8i1__9_6_1_10_3_7_7_1: +; VL_BW_DQ: # BB#0: +; VL_BW_DQ-NEXT: kmovb %edi, %k0 +; VL_BW_DQ-NEXT: movb $51, %al +; VL_BW_DQ-NEXT: kmovb %eax, %k1 +; VL_BW_DQ-NEXT: vpmovm2q %k1, %zmm0 +; VL_BW_DQ-NEXT: vpmovm2q %k0, %zmm1 +; VL_BW_DQ-NEXT: vmovdqa64 {{.*#+}} zmm2 = [9,6,1,0,3,7,7,1] +; VL_BW_DQ-NEXT: vpermt2q %zmm1, %zmm2, %zmm0 +; VL_BW_DQ-NEXT: vpsllq $63, %zmm0, %zmm0 +; VL_BW_DQ-NEXT: vpmovq2m %zmm0, %k0 +; VL_BW_DQ-NEXT: kmovb %k0, %eax +; VL_BW_DQ-NEXT: retq + %b = bitcast i8 %a to <8 x i1> + %c = shufflevector <8 x i1> , <8 x i1> %b, <8 x i32> + %c1 = bitcast <8 x i1>%c to i8 + ret i8 %c1 +} + +define i8 @shuf8i1_9_6_1_10_3_7_7_0_all_ones(<8 x i1> %a) { +; AVX512F-LABEL: shuf8i1_9_6_1_10_3_7_7_0_all_ones: +; AVX512F: # BB#0: +; AVX512F-NEXT: vpmovsxwq %xmm0, %zmm0 +; AVX512F-NEXT: vpsllq $63, %zmm0, %zmm0 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vpbroadcastq {{.*}}(%rip), %zmm0 {%k1} {z} +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [9,1,2,3,4,5,6,7] +; AVX512F-NEXT: vpbroadcastd {{.*}}(%rip), %zmm2 +; AVX512F-NEXT: vpermt2q %zmm0, %zmm1, %zmm2 +; AVX512F-NEXT: vpsllq $63, %zmm2, %zmm0 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0 +; AVX512F-NEXT: kmovw %k0, %eax +; AVX512F-NEXT: retq +; +; VL_BW_DQ-LABEL: shuf8i1_9_6_1_10_3_7_7_0_all_ones: +; VL_BW_DQ: # BB#0: +; VL_BW_DQ-NEXT: vpsllw $15, %xmm0, %xmm0 +; VL_BW_DQ-NEXT: vpmovw2m %xmm0, %k0 +; VL_BW_DQ-NEXT: vpmovm2q %k0, %zmm0 +; VL_BW_DQ-NEXT: vmovdqa64 {{.*#+}} zmm1 = [9,1,2,3,4,5,6,7] +; VL_BW_DQ-NEXT: vpbroadcastd {{.*}}(%rip), %zmm2 +; VL_BW_DQ-NEXT: vpermt2q %zmm0, %zmm1, %zmm2 +; VL_BW_DQ-NEXT: vpsllq $63, %zmm2, %zmm0 +; VL_BW_DQ-NEXT: vpmovq2m %zmm0, %k0 +; VL_BW_DQ-NEXT: kmovb %k0, %eax +; VL_BW_DQ-NEXT: retq + %c = shufflevector <8 x i1> , <8 x i1> %a, <8 x i32> + %c1 = bitcast <8 x i1>%c to i8 + ret i8 %c1 +} + + +define i16 @shuf16i1_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0(i16 %a) { +; AVX512F-LABEL: shuf16i1_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0: +; AVX512F: # BB#0: +; AVX512F-NEXT: kmovw %edi, %k1 +; AVX512F-NEXT: vpbroadcastd {{.*}}(%rip), %zmm0 {%k1} {z} +; AVX512F-NEXT: vpbroadcastd %xmm0, %zmm0 +; AVX512F-NEXT: vpslld $31, %zmm0, %zmm0 +; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k0 +; AVX512F-NEXT: kmovw %k0, %eax +; AVX512F-NEXT: retq +; +; VL_BW_DQ-LABEL: shuf16i1_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0_0: +; VL_BW_DQ: # BB#0: +; VL_BW_DQ-NEXT: kmovw %edi, %k0 +; VL_BW_DQ-NEXT: vpmovm2d %k0, %zmm0 +; VL_BW_DQ-NEXT: vpbroadcastd %xmm0, %zmm0 +; VL_BW_DQ-NEXT: vpslld $31, %zmm0, %zmm0 +; VL_BW_DQ-NEXT: vpmovd2m %zmm0, %k0 +; VL_BW_DQ-NEXT: kmovw %k0, %eax +; VL_BW_DQ-NEXT: retq + %b = bitcast i16 %a to <16 x i1> + %c = shufflevector < 16 x i1> %b, <16 x i1> undef, <16 x i32> zeroinitializer + %d = bitcast <16 x i1> %c to i16 + ret i16 %d +} + +define i64 @shuf64i1_zero(i64 %a) { +; AVX512F-LABEL: shuf64i1_zero: +; AVX512F: # BB#0: +; AVX512F-NEXT: pushq %rbp +; AVX512F-NEXT: .Ltmp0: +; AVX512F-NEXT: .cfi_def_cfa_offset 16 +; AVX512F-NEXT: .Ltmp1: +; AVX512F-NEXT: .cfi_offset %rbp, -16 +; AVX512F-NEXT: movq %rsp, %rbp +; AVX512F-NEXT: .Ltmp2: +; AVX512F-NEXT: .cfi_def_cfa_register %rbp +; AVX512F-NEXT: andq $-32, %rsp +; AVX512F-NEXT: subq $32, %rsp +; AVX512F-NEXT: movb $0, (%rsp) +; AVX512F-NEXT: movl (%rsp), %ecx +; AVX512F-NEXT: movq %rcx, %rax +; AVX512F-NEXT: shlq $32, %rax +; AVX512F-NEXT: orq %rcx, %rax +; AVX512F-NEXT: movq %rbp, %rsp +; AVX512F-NEXT: popq %rbp +; AVX512F-NEXT: retq +; +; VL_BW_DQ-LABEL: shuf64i1_zero: +; VL_BW_DQ: # BB#0: +; VL_BW_DQ-NEXT: kxorq %k0, %k0, %k0 +; VL_BW_DQ-NEXT: kmovq %k0, %rax +; VL_BW_DQ-NEXT: retq + %b = bitcast i64 %a to <64 x i1> + %c = shufflevector < 64 x i1> zeroinitializer, <64 x i1> undef, <64 x i32> zeroinitializer + %d = bitcast <64 x i1> %c to i64 + ret i64 %d +} diff --git a/test/CodeGen/X86/vector-trunc.ll b/test/CodeGen/X86/vector-trunc.ll index 2480e676cad0..8c02c5a5433f 100644 --- a/test/CodeGen/X86/vector-trunc.ll +++ b/test/CodeGen/X86/vector-trunc.ll @@ -1,38 +1,590 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=SSE --check-prefix=SSSE3 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx512bw | FileCheck %s --check-prefix=AVX512BW -define <4 x i32> @trunc2x2i64(<2 x i64> %a, <2 x i64> %b) { -; SSE2-LABEL: trunc2x2i64: +define <8 x i32> @trunc8i64_8i32(<8 x i64> %a) { +; SSE2-LABEL: trunc8i64_8i32: ; SSE2: # BB#0: # %entry ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3] +; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0] ; SSE2-NEXT: retq ; -; SSSE3-LABEL: trunc2x2i64: +; SSSE3-LABEL: trunc8i64_8i32: ; SSSE3: # BB#0: # %entry ; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] +; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3] +; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0] ; SSSE3-NEXT: retq ; -; SSE41-LABEL: trunc2x2i64: +; SSE41-LABEL: trunc8i64_8i32: ; SSE41: # BB#0: # %entry ; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2] ; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7] +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,2] +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3] +; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5,6,7] ; SSE41-NEXT: retq ; -; AVX-LABEL: trunc2x2i64: -; AVX: # BB#0: # %entry -; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,2] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7] -; AVX-NEXT: retq +; AVX1-LABEL: trunc8i64_8i32: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: trunc8i64_8i32: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = <0,2,4,6,u,u,u,u> +; AVX2-NEXT: vpermd %ymm0, %ymm2, %ymm0 +; AVX2-NEXT: vpermd %ymm1, %ymm2, %ymm1 +; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: trunc8i64_8i32: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmovqd %zmm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %0 = trunc <8 x i64> %a to <8 x i32> + ret <8 x i32> %0 +} + +define <8 x i16> @trunc8i64_8i16(<8 x i64> %a) { +; SSE2-LABEL: trunc8i64_8i16: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: pextrw $4, %xmm1, %eax +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3] +; SSE2-NEXT: pextrw $4, %xmm0, %ecx +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE2-NEXT: pextrw $4, %xmm3, %edx +; SSE2-NEXT: movd %edx, %xmm1 +; SSE2-NEXT: movd %eax, %xmm3 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3] +; SSE2-NEXT: pextrw $4, %xmm2, %eax +; SSE2-NEXT: movd %eax, %xmm1 +; SSE2-NEXT: movd %ecx, %xmm2 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: trunc8i64_8i16: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: pextrw $4, %xmm1, %eax +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3] +; SSSE3-NEXT: pextrw $4, %xmm0, %ecx +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSSE3-NEXT: pextrw $4, %xmm3, %edx +; SSSE3-NEXT: movd %edx, %xmm1 +; SSSE3-NEXT: movd %eax, %xmm3 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3] +; SSSE3-NEXT: pextrw $4, %xmm2, %eax +; SSSE3-NEXT: movd %eax, %xmm1 +; SSSE3-NEXT: movd %ecx, %xmm2 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: trunc8i64_8i16: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pxor %xmm4, %xmm4 +; SSE41-NEXT: pblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1,2,3],xmm3[4],xmm4[5,6,7] +; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1,2,3],xmm2[4],xmm4[5,6,7] +; SSE41-NEXT: packusdw %xmm3, %xmm2 +; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0],xmm4[1,2,3],xmm1[4],xmm4[5,6,7] +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0],xmm4[1,2,3],xmm0[4],xmm4[5,6,7] +; SSE41-NEXT: packusdw %xmm1, %xmm0 +; SSE41-NEXT: packusdw %xmm2, %xmm0 +; SSE41-NEXT: retq +; +; AVX1-LABEL: trunc8i64_8i16: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4],xmm3[5,6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm3[1,2,3],xmm1[4],xmm3[5,6,7] +; AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4],xmm3[5,6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1,2,3],xmm0[4],xmm3[5,6,7] +; AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: trunc8i64_8i16: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = <0,2,4,6,u,u,u,u> +; AVX2-NEXT: vpermd %ymm0, %ymm2, %ymm0 +; AVX2-NEXT: vpermd %ymm1, %ymm2, %ymm1 +; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,20,21,24,25,28,29],zero,zero,zero,zero,zero,zero,zero,zero +; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: trunc8i64_8i16: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmovqw %zmm0, %xmm0 +; AVX512BW-NEXT: retq +entry: + %0 = trunc <8 x i64> %a to <8 x i16> + ret <8 x i16> %0 +} + +define void @trunc8i64_8i8(<8 x i64> %a) { +; SSE-LABEL: trunc8i64_8i8: +; SSE: # BB#0: # %entry +; SSE-NEXT: movdqa {{.*#+}} xmm4 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0] +; SSE-NEXT: pand %xmm4, %xmm3 +; SSE-NEXT: pand %xmm4, %xmm2 +; SSE-NEXT: packuswb %xmm3, %xmm2 +; SSE-NEXT: pand %xmm4, %xmm1 +; SSE-NEXT: pand %xmm4, %xmm0 +; SSE-NEXT: packuswb %xmm1, %xmm0 +; SSE-NEXT: packuswb %xmm2, %xmm0 +; SSE-NEXT: packuswb %xmm0, %xmm0 +; SSE-NEXT: movq %xmm0, (%rax) +; SSE-NEXT: retq +; +; AVX1-LABEL: trunc8i64_8i8: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0] +; AVX1-NEXT: vandps %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vandps %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpackuswb %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vandps %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vandps %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpackuswb %xmm0, %xmm0, %xmm0 +; AVX1-NEXT: vmovq %xmm0, (%rax) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: trunc8i64_8i8: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = <0,2,4,6,u,u,u,u> +; AVX2-NEXT: vpermd %ymm0, %ymm2, %ymm0 +; AVX2-NEXT: vpermd %ymm1, %ymm2, %ymm1 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u> +; AVX2-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; AVX2-NEXT: vpshufb %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; AVX2-NEXT: vmovq %xmm0, (%rax) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: trunc8i64_8i8: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmovqb %zmm0, (%rax) +; AVX512BW-NEXT: retq +entry: + %0 = trunc <8 x i64> %a to <8 x i8> + store <8 x i8> %0, <8 x i8>* undef, align 4 + ret void +} + +define <8 x i16> @trunc8i32_8i16(<8 x i32> %a) { +; SSE2-LABEL: trunc8i32_8i16: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: pslld $16, %xmm1 +; SSE2-NEXT: psrad $16, %xmm1 +; SSE2-NEXT: pslld $16, %xmm0 +; SSE2-NEXT: psrad $16, %xmm0 +; SSE2-NEXT: packssdw %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSSE3-LABEL: trunc8i32_8i16: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] +; SSSE3-NEXT: pshufb %xmm2, %xmm1 +; SSSE3-NEXT: pshufb %xmm2, %xmm0 +; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: trunc8i32_8i16: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] +; SSE41-NEXT: pshufb %xmm2, %xmm1 +; SSE41-NEXT: pshufb %xmm2, %xmm0 +; SSE41-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE41-NEXT: retq +; +; AVX1-LABEL: trunc8i32_8i16: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] +; AVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: trunc8i32_8i16: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,20,21,24,25,28,29],zero,zero,zero,zero,zero,zero,zero,zero +; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: trunc8i32_8i16: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmovdw %zmm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %0 = trunc <8 x i32> %a to <8 x i16> + ret <8 x i16> %0 +} +define void @trunc8i32_8i8(<8 x i32> %a) { +; SSE2-LABEL: trunc8i32_8i8: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0] +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: packuswb %xmm1, %xmm0 +; SSE2-NEXT: packuswb %xmm0, %xmm0 +; SSE2-NEXT: movq %xmm0, (%rax) +; SSE2-NEXT: retq +; +; SSSE3-LABEL: trunc8i32_8i8: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u> +; SSSE3-NEXT: pshufb %xmm2, %xmm1 +; SSSE3-NEXT: pshufb %xmm2, %xmm0 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSSE3-NEXT: movq %xmm0, (%rax) +; SSSE3-NEXT: retq +; +; SSE41-LABEL: trunc8i32_8i8: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u> +; SSE41-NEXT: pshufb %xmm2, %xmm1 +; SSE41-NEXT: pshufb %xmm2, %xmm0 +; SSE41-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE41-NEXT: movq %xmm0, (%rax) +; SSE41-NEXT: retq +; +; AVX1-LABEL: trunc8i32_8i8: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u> +; AVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; AVX1-NEXT: vmovq %xmm0, (%rax) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: trunc8i32_8i8: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,20,21,24,25,28,29],zero,zero,zero,zero,zero,zero,zero,zero +; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] +; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u] +; AVX2-NEXT: vmovq %xmm0, (%rax) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: trunc8i32_8i8: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmovdw %zmm0, %ymm0 +; AVX512BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u] +; AVX512BW-NEXT: vmovq %xmm0, (%rax) +; AVX512BW-NEXT: retq +entry: + %0 = trunc <8 x i32> %a to <8 x i8> + store <8 x i8> %0, <8 x i8>* undef, align 4 + ret void +} +define void @trunc16i32_16i8(<16 x i32> %a) { +; SSE-LABEL: trunc16i32_16i8: +; SSE: # BB#0: # %entry +; SSE-NEXT: movdqa {{.*#+}} xmm4 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0] +; SSE-NEXT: pand %xmm4, %xmm3 +; SSE-NEXT: pand %xmm4, %xmm2 +; SSE-NEXT: packuswb %xmm3, %xmm2 +; SSE-NEXT: pand %xmm4, %xmm1 +; SSE-NEXT: pand %xmm4, %xmm0 +; SSE-NEXT: packuswb %xmm1, %xmm0 +; SSE-NEXT: packuswb %xmm2, %xmm0 +; SSE-NEXT: movdqu %xmm0, (%rax) +; SSE-NEXT: retq +; +; AVX1-LABEL: trunc16i32_16i8: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0] +; AVX1-NEXT: vandps %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vandps %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpackuswb %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vandps %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vandps %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vmovdqu %xmm0, (%rax) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: trunc16i32_16i8: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128,0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128] +; AVX2-NEXT: vpshufb %ymm2, %ymm1, %ymm1 +; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3] +; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u> +; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1 +; AVX2-NEXT: vpshufb %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] +; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm0 +; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX2-NEXT: vmovdqu %xmm0, (%rax) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: trunc16i32_16i8: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmovdb %zmm0, (%rax) +; AVX512BW-NEXT: retq +entry: + %0 = trunc <16 x i32> %a to <16 x i8> + store <16 x i8> %0, <16 x i8>* undef, align 4 + ret void +} + +define <8 x i32> @trunc2x4i64_8i32(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: trunc2x4i64_8i32: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3] +; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: trunc2x4i64_8i32: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] +; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3] +; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: trunc2x4i64_8i32: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2] +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7] +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,2] +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3] +; SSE41-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5,6,7] +; SSE41-NEXT: retq +; +; AVX1-LABEL: trunc2x4i64_8i32: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: trunc2x4i64_8i32: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = <0,2,4,6,u,u,u,u> +; AVX2-NEXT: vpermd %ymm0, %ymm2, %ymm0 +; AVX2-NEXT: vpermd %ymm1, %ymm2, %ymm1 +; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: trunc2x4i64_8i32: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmovqd %zmm0, %ymm0 +; AVX512BW-NEXT: vpmovqd %zmm1, %ymm1 +; AVX512BW-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %0 = trunc <4 x i64> %a to <4 x i32> + %1 = trunc <4 x i64> %b to <4 x i32> + %2 = shufflevector <4 x i32> %0, <4 x i32> %1, <8 x i32> + ret <8 x i32> %2 +} + +define <8 x i16> @trunc2x4i64_8i16(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: trunc2x4i64_8i16: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: pextrw $4, %xmm1, %eax +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3] +; SSE2-NEXT: pextrw $4, %xmm0, %ecx +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE2-NEXT: pextrw $4, %xmm3, %edx +; SSE2-NEXT: movd %edx, %xmm1 +; SSE2-NEXT: movd %eax, %xmm3 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3] +; SSE2-NEXT: pextrw $4, %xmm2, %eax +; SSE2-NEXT: movd %eax, %xmm1 +; SSE2-NEXT: movd %ecx, %xmm2 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: trunc2x4i64_8i16: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: pextrw $4, %xmm1, %eax +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3] +; SSSE3-NEXT: pextrw $4, %xmm0, %ecx +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSSE3-NEXT: pextrw $4, %xmm3, %edx +; SSSE3-NEXT: movd %edx, %xmm1 +; SSSE3-NEXT: movd %eax, %xmm3 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3] +; SSSE3-NEXT: pextrw $4, %xmm2, %eax +; SSSE3-NEXT: movd %eax, %xmm1 +; SSSE3-NEXT: movd %ecx, %xmm2 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: trunc2x4i64_8i16: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pextrw $4, %xmm0, %eax +; SSE41-NEXT: pinsrw $1, %eax, %xmm0 +; SSE41-NEXT: movd %xmm1, %eax +; SSE41-NEXT: pinsrw $2, %eax, %xmm0 +; SSE41-NEXT: pextrw $4, %xmm1, %eax +; SSE41-NEXT: pinsrw $3, %eax, %xmm0 +; SSE41-NEXT: movd %xmm2, %eax +; SSE41-NEXT: pinsrw $4, %eax, %xmm0 +; SSE41-NEXT: pextrw $4, %xmm2, %eax +; SSE41-NEXT: pinsrw $5, %eax, %xmm0 +; SSE41-NEXT: movd %xmm3, %eax +; SSE41-NEXT: pinsrw $6, %eax, %xmm0 +; SSE41-NEXT: pextrw $4, %xmm3, %eax +; SSE41-NEXT: pinsrw $7, %eax, %xmm0 +; SSE41-NEXT: retq +; +; AVX1-LABEL: trunc2x4i64_8i16: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] +; AVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: trunc2x4i64_8i16: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = <0,2,4,6,u,u,u,u> +; AVX2-NEXT: vpermd %ymm0, %ymm2, %ymm0 +; AVX2-NEXT: vpermd %ymm1, %ymm2, %ymm1 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] +; AVX2-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; AVX2-NEXT: vpshufb %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: trunc2x4i64_8i16: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmovqd %zmm0, %ymm0 +; AVX512BW-NEXT: vpmovqd %zmm1, %ymm1 +; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] +; AVX512BW-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; AVX512BW-NEXT: vpshufb %xmm2, %xmm0, %xmm0 +; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX512BW-NEXT: retq +entry: + %0 = trunc <4 x i64> %a to <4 x i16> + %1 = trunc <4 x i64> %b to <4 x i16> + %2 = shufflevector <4 x i16> %0, <4 x i16> %1, <8 x i32> + ret <8 x i16> %2 +} + +define <4 x i32> @trunc2x2i64_4i32(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: trunc2x2i64_4i32: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: trunc2x2i64_4i32: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: trunc2x2i64_4i32: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2] +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7] +; SSE41-NEXT: retq +; +; AVX1-LABEL: trunc2x2i64_4i32: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,2] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7] +; AVX1-NEXT: retq +; +; AVX2-LABEL: trunc2x2i64_4i32: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,2] +; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3] +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: trunc2x2i64_4i32: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,2] +; AVX512BW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3] +; AVX512BW-NEXT: retq entry: %0 = trunc <2 x i64> %a to <2 x i32> %1 = trunc <2 x i64> %b to <2 x i32> @@ -40,28 +592,32 @@ entry: ret <4 x i32> %2 } -define i64 @trunc2i64(<2 x i64> %inval) { -; SSE-LABEL: trunc2i64: +define i64 @trunc2i64_i64(<2 x i64> %inval) { +; SSE-LABEL: trunc2i64_i64: ; SSE: # BB#0: # %entry ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: movd %xmm0, %rax ; SSE-NEXT: retq ; -; AVX-LABEL: trunc2i64: +; AVX-LABEL: trunc2i64_i64: ; AVX: # BB#0: # %entry ; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; AVX-NEXT: vmovq %xmm0, %rax ; AVX-NEXT: retq - - +; +; AVX512BW-LABEL: trunc2i64_i64: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; AVX512BW-NEXT: vmovq %xmm0, %rax +; AVX512BW-NEXT: retq entry: %0 = trunc <2 x i64> %inval to <2 x i32> %1 = bitcast <2 x i32> %0 to i64 ret i64 %1 } -define <8 x i16> @trunc2x4i32(<4 x i32> %a, <4 x i32> %b) { -; SSE2-LABEL: trunc2x4i32: +define <8 x i16> @trunc2x4i32_8i16(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: trunc2x4i32_8i16: ; SSE2: # BB#0: # %entry ; SSE2-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7] ; SSE2-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7] @@ -72,7 +628,7 @@ define <8 x i16> @trunc2x4i32(<4 x i32> %a, <4 x i32> %b) { ; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] ; SSE2-NEXT: retq ; -; SSSE3-LABEL: trunc2x4i32: +; SSSE3-LABEL: trunc2x4i32_8i16: ; SSSE3: # BB#0: # %entry ; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] ; SSSE3-NEXT: pshufb %xmm2, %xmm1 @@ -80,7 +636,7 @@ define <8 x i16> @trunc2x4i32(<4 x i32> %a, <4 x i32> %b) { ; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] ; SSSE3-NEXT: retq ; -; SSE41-LABEL: trunc2x4i32: +; SSE41-LABEL: trunc2x4i32_8i16: ; SSE41: # BB#0: # %entry ; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] ; SSE41-NEXT: pshufb %xmm2, %xmm1 @@ -88,17 +644,21 @@ define <8 x i16> @trunc2x4i32(<4 x i32> %a, <4 x i32> %b) { ; SSE41-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] ; SSE41-NEXT: retq ; -; AVX-LABEL: trunc2x4i32: +; AVX-LABEL: trunc2x4i32_8i16: ; AVX: # BB#0: # %entry ; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] ; AVX-NEXT: vpshufb %xmm2, %xmm1, %xmm1 ; AVX-NEXT: vpshufb %xmm2, %xmm0, %xmm0 ; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] ; AVX-NEXT: retq - - - - +; +; AVX512BW-LABEL: trunc2x4i32_8i16: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] +; AVX512BW-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; AVX512BW-NEXT: vpshufb %xmm2, %xmm0, %xmm0 +; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX512BW-NEXT: retq entry: %0 = trunc <4 x i32> %a to <4 x i16> %1 = trunc <4 x i32> %b to <4 x i16> @@ -107,8 +667,8 @@ entry: } ; PR15524 http://llvm.org/bugs/show_bug.cgi?id=15524 -define i64 @trunc4i32(<4 x i32> %inval) { -; SSE2-LABEL: trunc4i32: +define i64 @trunc4i32_i64(<4 x i32> %inval) { +; SSE2-LABEL: trunc4i32_i64: ; SSE2: # BB#0: # %entry ; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] ; SSE2-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7] @@ -116,35 +676,37 @@ define i64 @trunc4i32(<4 x i32> %inval) { ; SSE2-NEXT: movd %xmm0, %rax ; SSE2-NEXT: retq ; -; SSSE3-LABEL: trunc4i32: +; SSSE3-LABEL: trunc4i32_i64: ; SSSE3: # BB#0: # %entry ; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] ; SSSE3-NEXT: movd %xmm0, %rax ; SSSE3-NEXT: retq ; -; SSE41-LABEL: trunc4i32: +; SSE41-LABEL: trunc4i32_i64: ; SSE41: # BB#0: # %entry ; SSE41-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] ; SSE41-NEXT: movd %xmm0, %rax ; SSE41-NEXT: retq ; -; AVX-LABEL: trunc4i32: +; AVX-LABEL: trunc4i32_i64: ; AVX: # BB#0: # %entry ; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] ; AVX-NEXT: vmovq %xmm0, %rax ; AVX-NEXT: retq - - - - +; +; AVX512BW-LABEL: trunc4i32_i64: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] +; AVX512BW-NEXT: vmovq %xmm0, %rax +; AVX512BW-NEXT: retq entry: %0 = trunc <4 x i32> %inval to <4 x i16> %1 = bitcast <4 x i16> %0 to i64 ret i64 %1 } -define <16 x i8> @trunc2x8i16(<8 x i16> %a, <8 x i16> %b) { -; SSE2-LABEL: trunc2x8i16: +define <16 x i8> @trunc2x8i16_16i8(<8 x i16> %a, <8 x i16> %b) { +; SSE2-LABEL: trunc2x8i16_16i8: ; SSE2: # BB#0: # %entry ; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255] ; SSE2-NEXT: pand %xmm2, %xmm1 @@ -152,7 +714,7 @@ define <16 x i8> @trunc2x8i16(<8 x i16> %a, <8 x i16> %b) { ; SSE2-NEXT: packuswb %xmm1, %xmm0 ; SSE2-NEXT: retq ; -; SSSE3-LABEL: trunc2x8i16: +; SSSE3-LABEL: trunc2x8i16_16i8: ; SSSE3: # BB#0: # %entry ; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u> ; SSSE3-NEXT: pshufb %xmm2, %xmm1 @@ -160,7 +722,7 @@ define <16 x i8> @trunc2x8i16(<8 x i16> %a, <8 x i16> %b) { ; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] ; SSSE3-NEXT: retq ; -; SSE41-LABEL: trunc2x8i16: +; SSE41-LABEL: trunc2x8i16_16i8: ; SSE41: # BB#0: # %entry ; SSE41-NEXT: movdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u> ; SSE41-NEXT: pshufb %xmm2, %xmm1 @@ -168,17 +730,21 @@ define <16 x i8> @trunc2x8i16(<8 x i16> %a, <8 x i16> %b) { ; SSE41-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] ; SSE41-NEXT: retq ; -; AVX-LABEL: trunc2x8i16: +; AVX-LABEL: trunc2x8i16_16i8: ; AVX: # BB#0: # %entry ; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u> ; AVX-NEXT: vpshufb %xmm2, %xmm1, %xmm1 ; AVX-NEXT: vpshufb %xmm2, %xmm0, %xmm0 ; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] ; AVX-NEXT: retq - - - - +; +; AVX512BW-LABEL: trunc2x8i16_16i8: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u> +; AVX512BW-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; AVX512BW-NEXT: vpshufb %xmm2, %xmm0, %xmm0 +; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX512BW-NEXT: retq entry: %0 = trunc <8 x i16> %a to <8 x i8> %1 = trunc <8 x i16> %b to <8 x i8> @@ -187,51 +753,58 @@ entry: } ; PR15524 http://llvm.org/bugs/show_bug.cgi?id=15524 -define i64 @trunc8i16(<8 x i16> %inval) { -; SSE2-LABEL: trunc8i16: +define i64 @trunc8i16_i64(<8 x i16> %inval) { +; SSE2-LABEL: trunc8i16_i64: ; SSE2: # BB#0: # %entry ; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 ; SSE2-NEXT: packuswb %xmm0, %xmm0 ; SSE2-NEXT: movd %xmm0, %rax ; SSE2-NEXT: retq ; -; SSSE3-LABEL: trunc8i16: +; SSSE3-LABEL: trunc8i16_i64: ; SSSE3: # BB#0: # %entry ; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u] ; SSSE3-NEXT: movd %xmm0, %rax ; SSSE3-NEXT: retq ; -; SSE41-LABEL: trunc8i16: +; SSE41-LABEL: trunc8i16_i64: ; SSE41: # BB#0: # %entry ; SSE41-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u] ; SSE41-NEXT: movd %xmm0, %rax ; SSE41-NEXT: retq ; -; AVX-LABEL: trunc8i16: +; AVX-LABEL: trunc8i16_i64: ; AVX: # BB#0: # %entry ; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u] ; AVX-NEXT: vmovq %xmm0, %rax ; AVX-NEXT: retq - - - - +; +; AVX512BW-LABEL: trunc8i16_i64: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u] +; AVX512BW-NEXT: vmovq %xmm0, %rax +; AVX512BW-NEXT: retq entry: %0 = trunc <8 x i16> %inval to <8 x i8> %1 = bitcast <8 x i8> %0 to i64 ret i64 %1 } -define <16 x i8> @trunc16i64_const() { -; SSE-LABEL: trunc16i64_const: +define <16 x i8> @trunc16i64_16i8_const() { +; SSE-LABEL: trunc16i64_16i8_const: ; SSE: # BB#0: # %entry ; SSE-NEXT: xorps %xmm0, %xmm0 ; SSE-NEXT: retq ; -; AVX-LABEL: trunc16i64_const: +; AVX-LABEL: trunc16i64_16i8_const: ; AVX: # BB#0: # %entry ; AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0 ; AVX-NEXT: retq +; +; AVX512BW-LABEL: trunc16i64_16i8_const: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vxorps %xmm0, %xmm0, %xmm0 +; AVX512BW-NEXT: retq entry: %0 = trunc <16 x i64> zeroinitializer to <16 x i8> diff --git a/test/CodeGen/X86/vector-tzcnt-128.ll b/test/CodeGen/X86/vector-tzcnt-128.ll index 422fe052d38b..f1714d4845de 100644 --- a/test/CodeGen/X86/vector-tzcnt-128.ll +++ b/test/CodeGen/X86/vector-tzcnt-128.ll @@ -1,13 +1,12 @@ -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE3 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE3 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 -target triple = "x86_64-unknown-unknown" - -define <2 x i64> @testv2i64(<2 x i64> %in) { +define <2 x i64> @testv2i64(<2 x i64> %in) nounwind { ; SSE2-LABEL: testv2i64: ; SSE2: # BB#0: ; SSE2-NEXT: movd %xmm0, %rax @@ -87,7 +86,7 @@ define <2 x i64> @testv2i64(<2 x i64> %in) { ret <2 x i64> %out } -define <2 x i64> @testv2i64u(<2 x i64> %in) { +define <2 x i64> @testv2i64u(<2 x i64> %in) nounwind { ; SSE2-LABEL: testv2i64u: ; SSE2: # BB#0: ; SSE2-NEXT: movd %xmm0, %rax @@ -152,1521 +151,755 @@ define <2 x i64> @testv2i64u(<2 x i64> %in) { ret <2 x i64> %out } -define <4 x i32> @testv4i32(<4 x i32> %in) { +define <4 x i32> @testv4i32(<4 x i32> %in) nounwind { ; SSE2-LABEL: testv4i32: ; SSE2: # BB#0: -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3] -; SSE2-NEXT: movd %xmm1, %eax -; SSE2-NEXT: bsfl %eax, %eax -; SSE2-NEXT: movl $32, %ecx -; SSE2-NEXT: cmovel %ecx, %eax -; SSE2-NEXT: movd %eax, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3] -; SSE2-NEXT: movd %xmm2, %eax -; SSE2-NEXT: bsfl %eax, %eax -; SSE2-NEXT: cmovel %ecx, %eax -; SSE2-NEXT: movd %eax, %xmm2 -; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; SSE2-NEXT: movd %xmm0, %eax -; SSE2-NEXT: bsfl %eax, %eax -; SSE2-NEXT: cmovel %ecx, %eax -; SSE2-NEXT: movd %eax, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE2-NEXT: movd %xmm0, %eax -; SSE2-NEXT: bsfl %eax, %eax -; SSE2-NEXT: cmovel %ecx, %eax -; SSE2-NEXT: movd %eax, %xmm0 -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] -; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: psubd %xmm0, %xmm2 +; SSE2-NEXT: pand %xmm0, %xmm2 +; SSE2-NEXT: psubd {{.*}}(%rip), %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: psrld $1, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: psubd %xmm0, %xmm2 +; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [858993459,858993459,858993459,858993459] +; SSE2-NEXT: movdqa %xmm2, %xmm3 +; SSE2-NEXT: pand %xmm0, %xmm3 +; SSE2-NEXT: psrld $2, %xmm2 +; SSE2-NEXT: pand %xmm0, %xmm2 +; SSE2-NEXT: paddd %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: psrld $4, %xmm0 +; SSE2-NEXT: paddd %xmm2, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSE2-NEXT: psadbw %xmm1, %xmm2 +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE2-NEXT: psadbw %xmm1, %xmm0 +; SSE2-NEXT: packuswb %xmm2, %xmm0 ; SSE2-NEXT: retq ; ; SSE3-LABEL: testv4i32: ; SSE3: # BB#0: -; SSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3] -; SSE3-NEXT: movd %xmm1, %eax -; SSE3-NEXT: bsfl %eax, %eax -; SSE3-NEXT: movl $32, %ecx -; SSE3-NEXT: cmovel %ecx, %eax -; SSE3-NEXT: movd %eax, %xmm1 -; SSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3] -; SSE3-NEXT: movd %xmm2, %eax -; SSE3-NEXT: bsfl %eax, %eax -; SSE3-NEXT: cmovel %ecx, %eax -; SSE3-NEXT: movd %eax, %xmm2 -; SSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; SSE3-NEXT: movd %xmm0, %eax -; SSE3-NEXT: bsfl %eax, %eax -; SSE3-NEXT: cmovel %ecx, %eax -; SSE3-NEXT: movd %eax, %xmm1 -; SSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE3-NEXT: movd %xmm0, %eax -; SSE3-NEXT: bsfl %eax, %eax -; SSE3-NEXT: cmovel %ecx, %eax -; SSE3-NEXT: movd %eax, %xmm0 -; SSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] -; SSE3-NEXT: movdqa %xmm1, %xmm0 +; SSE3-NEXT: pxor %xmm1, %xmm1 +; SSE3-NEXT: pxor %xmm2, %xmm2 +; SSE3-NEXT: psubd %xmm0, %xmm2 +; SSE3-NEXT: pand %xmm0, %xmm2 +; SSE3-NEXT: psubd {{.*}}(%rip), %xmm2 +; SSE3-NEXT: movdqa %xmm2, %xmm0 +; SSE3-NEXT: psrld $1, %xmm0 +; SSE3-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE3-NEXT: psubd %xmm0, %xmm2 +; SSE3-NEXT: movdqa {{.*#+}} xmm0 = [858993459,858993459,858993459,858993459] +; SSE3-NEXT: movdqa %xmm2, %xmm3 +; SSE3-NEXT: pand %xmm0, %xmm3 +; SSE3-NEXT: psrld $2, %xmm2 +; SSE3-NEXT: pand %xmm0, %xmm2 +; SSE3-NEXT: paddd %xmm3, %xmm2 +; SSE3-NEXT: movdqa %xmm2, %xmm0 +; SSE3-NEXT: psrld $4, %xmm0 +; SSE3-NEXT: paddd %xmm2, %xmm0 +; SSE3-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE3-NEXT: movdqa %xmm0, %xmm2 +; SSE3-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSE3-NEXT: psadbw %xmm1, %xmm2 +; SSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE3-NEXT: psadbw %xmm1, %xmm0 +; SSE3-NEXT: packuswb %xmm2, %xmm0 ; SSE3-NEXT: retq ; ; SSSE3-LABEL: testv4i32: ; SSSE3: # BB#0: -; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3] -; SSSE3-NEXT: movd %xmm1, %eax -; SSSE3-NEXT: bsfl %eax, %eax -; SSSE3-NEXT: movl $32, %ecx -; SSSE3-NEXT: cmovel %ecx, %eax -; SSSE3-NEXT: movd %eax, %xmm1 -; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3] -; SSSE3-NEXT: movd %xmm2, %eax -; SSSE3-NEXT: bsfl %eax, %eax -; SSSE3-NEXT: cmovel %ecx, %eax -; SSSE3-NEXT: movd %eax, %xmm2 -; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; SSSE3-NEXT: movd %xmm0, %eax -; SSSE3-NEXT: bsfl %eax, %eax -; SSSE3-NEXT: cmovel %ecx, %eax -; SSSE3-NEXT: movd %eax, %xmm1 -; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSSE3-NEXT: movd %xmm0, %eax -; SSSE3-NEXT: bsfl %eax, %eax -; SSSE3-NEXT: cmovel %ecx, %eax -; SSSE3-NEXT: movd %eax, %xmm0 -; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] -; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: psubd %xmm0, %xmm2 +; SSSE3-NEXT: pand %xmm0, %xmm2 +; SSSE3-NEXT: psubd {{.*}}(%rip), %xmm2 +; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSSE3-NEXT: movdqa %xmm2, %xmm4 +; SSSE3-NEXT: pand %xmm3, %xmm4 +; SSSE3-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; SSSE3-NEXT: movdqa %xmm0, %xmm5 +; SSSE3-NEXT: pshufb %xmm4, %xmm5 +; SSSE3-NEXT: psrlw $4, %xmm2 +; SSSE3-NEXT: pand %xmm3, %xmm2 +; SSSE3-NEXT: pshufb %xmm2, %xmm0 +; SSSE3-NEXT: paddb %xmm5, %xmm0 +; SSSE3-NEXT: movdqa %xmm0, %xmm2 +; SSSE3-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSSE3-NEXT: psadbw %xmm1, %xmm2 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSSE3-NEXT: psadbw %xmm1, %xmm0 +; SSSE3-NEXT: packuswb %xmm2, %xmm0 ; SSSE3-NEXT: retq ; ; SSE41-LABEL: testv4i32: ; SSE41: # BB#0: -; SSE41-NEXT: pextrd $1, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: movl $32, %ecx -; SSE41-NEXT: cmovel %ecx, %eax -; SSE41-NEXT: movd %xmm0, %edx -; SSE41-NEXT: bsfl %edx, %edx -; SSE41-NEXT: cmovel %ecx, %edx -; SSE41-NEXT: movd %edx, %xmm1 -; SSE41-NEXT: pinsrd $1, %eax, %xmm1 -; SSE41-NEXT: pextrd $2, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: cmovel %ecx, %eax -; SSE41-NEXT: pinsrd $2, %eax, %xmm1 -; SSE41-NEXT: pextrd $3, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: cmovel %ecx, %eax -; SSE41-NEXT: pinsrd $3, %eax, %xmm1 -; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pxor %xmm1, %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: psubd %xmm0, %xmm2 +; SSE41-NEXT: pand %xmm0, %xmm2 +; SSE41-NEXT: psubd {{.*}}(%rip), %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSE41-NEXT: movdqa %xmm2, %xmm4 +; SSE41-NEXT: pand %xmm3, %xmm4 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pshufb %xmm4, %xmm5 +; SSE41-NEXT: psrlw $4, %xmm2 +; SSE41-NEXT: pand %xmm3, %xmm2 +; SSE41-NEXT: pshufb %xmm2, %xmm0 +; SSE41-NEXT: paddb %xmm5, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSE41-NEXT: psadbw %xmm1, %xmm2 +; SSE41-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE41-NEXT: psadbw %xmm1, %xmm0 +; SSE41-NEXT: packuswb %xmm2, %xmm0 ; SSE41-NEXT: retq ; -; AVX-LABEL: testv4i32: -; AVX: # BB#0: -; AVX-NEXT: vpextrd $1, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: movl $32, %ecx -; AVX-NEXT: cmovel %ecx, %eax -; AVX-NEXT: vmovd %xmm0, %edx -; AVX-NEXT: bsfl %edx, %edx -; AVX-NEXT: cmovel %ecx, %edx -; AVX-NEXT: vmovd %edx, %xmm1 -; AVX-NEXT: vpinsrd $1, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrd $2, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: cmovel %ecx, %eax -; AVX-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrd $3, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: cmovel %ecx, %eax -; AVX-NEXT: vpinsrd $3, %eax, %xmm1, %xmm0 -; AVX-NEXT: retq +; AVX1-LABEL: testv4i32: +; AVX1: # BB#0: +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpsubd %xmm0, %xmm1, %xmm2 +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpsubd {{.*}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0 +; AVX1-NEXT: vpaddb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX1-NEXT: vpsadbw %xmm1, %xmm2, %xmm2 +; AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: testv4i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX2-NEXT: vpsubd %xmm0, %xmm1, %xmm2 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %xmm2 +; AVX2-NEXT: vpsubd %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpshufb %xmm0, %xmm4, %xmm0 +; AVX2-NEXT: vpaddb %xmm3, %xmm0, %xmm0 +; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX2-NEXT: vpsadbw %xmm1, %xmm2, %xmm2 +; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpackuswb %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: retq %out = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %in, i1 0) ret <4 x i32> %out } -define <4 x i32> @testv4i32u(<4 x i32> %in) { +define <4 x i32> @testv4i32u(<4 x i32> %in) nounwind { ; SSE2-LABEL: testv4i32u: ; SSE2: # BB#0: -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3] -; SSE2-NEXT: movd %xmm1, %eax -; SSE2-NEXT: bsfl %eax, %eax -; SSE2-NEXT: movd %eax, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3] -; SSE2-NEXT: movd %xmm2, %eax -; SSE2-NEXT: bsfl %eax, %eax -; SSE2-NEXT: movd %eax, %xmm2 -; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; SSE2-NEXT: movd %xmm0, %eax -; SSE2-NEXT: bsfl %eax, %eax -; SSE2-NEXT: movd %eax, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE2-NEXT: movd %xmm0, %eax -; SSE2-NEXT: bsfl %eax, %eax -; SSE2-NEXT: movd %eax, %xmm0 -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] -; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: psubd %xmm0, %xmm2 +; SSE2-NEXT: pand %xmm0, %xmm2 +; SSE2-NEXT: psubd {{.*}}(%rip), %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: psrld $1, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: psubd %xmm0, %xmm2 +; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [858993459,858993459,858993459,858993459] +; SSE2-NEXT: movdqa %xmm2, %xmm3 +; SSE2-NEXT: pand %xmm0, %xmm3 +; SSE2-NEXT: psrld $2, %xmm2 +; SSE2-NEXT: pand %xmm0, %xmm2 +; SSE2-NEXT: paddd %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: psrld $4, %xmm0 +; SSE2-NEXT: paddd %xmm2, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSE2-NEXT: psadbw %xmm1, %xmm2 +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE2-NEXT: psadbw %xmm1, %xmm0 +; SSE2-NEXT: packuswb %xmm2, %xmm0 ; SSE2-NEXT: retq ; ; SSE3-LABEL: testv4i32u: ; SSE3: # BB#0: -; SSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3] -; SSE3-NEXT: movd %xmm1, %eax -; SSE3-NEXT: bsfl %eax, %eax -; SSE3-NEXT: movd %eax, %xmm1 -; SSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3] -; SSE3-NEXT: movd %xmm2, %eax -; SSE3-NEXT: bsfl %eax, %eax -; SSE3-NEXT: movd %eax, %xmm2 -; SSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; SSE3-NEXT: movd %xmm0, %eax -; SSE3-NEXT: bsfl %eax, %eax -; SSE3-NEXT: movd %eax, %xmm1 -; SSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE3-NEXT: movd %xmm0, %eax -; SSE3-NEXT: bsfl %eax, %eax -; SSE3-NEXT: movd %eax, %xmm0 -; SSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] -; SSE3-NEXT: movdqa %xmm1, %xmm0 +; SSE3-NEXT: pxor %xmm1, %xmm1 +; SSE3-NEXT: pxor %xmm2, %xmm2 +; SSE3-NEXT: psubd %xmm0, %xmm2 +; SSE3-NEXT: pand %xmm0, %xmm2 +; SSE3-NEXT: psubd {{.*}}(%rip), %xmm2 +; SSE3-NEXT: movdqa %xmm2, %xmm0 +; SSE3-NEXT: psrld $1, %xmm0 +; SSE3-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE3-NEXT: psubd %xmm0, %xmm2 +; SSE3-NEXT: movdqa {{.*#+}} xmm0 = [858993459,858993459,858993459,858993459] +; SSE3-NEXT: movdqa %xmm2, %xmm3 +; SSE3-NEXT: pand %xmm0, %xmm3 +; SSE3-NEXT: psrld $2, %xmm2 +; SSE3-NEXT: pand %xmm0, %xmm2 +; SSE3-NEXT: paddd %xmm3, %xmm2 +; SSE3-NEXT: movdqa %xmm2, %xmm0 +; SSE3-NEXT: psrld $4, %xmm0 +; SSE3-NEXT: paddd %xmm2, %xmm0 +; SSE3-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE3-NEXT: movdqa %xmm0, %xmm2 +; SSE3-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSE3-NEXT: psadbw %xmm1, %xmm2 +; SSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE3-NEXT: psadbw %xmm1, %xmm0 +; SSE3-NEXT: packuswb %xmm2, %xmm0 ; SSE3-NEXT: retq ; ; SSSE3-LABEL: testv4i32u: ; SSSE3: # BB#0: -; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3] -; SSSE3-NEXT: movd %xmm1, %eax -; SSSE3-NEXT: bsfl %eax, %eax -; SSSE3-NEXT: movd %eax, %xmm1 -; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3] -; SSSE3-NEXT: movd %xmm2, %eax -; SSSE3-NEXT: bsfl %eax, %eax -; SSSE3-NEXT: movd %eax, %xmm2 -; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; SSSE3-NEXT: movd %xmm0, %eax -; SSSE3-NEXT: bsfl %eax, %eax -; SSSE3-NEXT: movd %eax, %xmm1 -; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSSE3-NEXT: movd %xmm0, %eax -; SSSE3-NEXT: bsfl %eax, %eax -; SSSE3-NEXT: movd %eax, %xmm0 -; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] -; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: psubd %xmm0, %xmm2 +; SSSE3-NEXT: pand %xmm0, %xmm2 +; SSSE3-NEXT: psubd {{.*}}(%rip), %xmm2 +; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSSE3-NEXT: movdqa %xmm2, %xmm4 +; SSSE3-NEXT: pand %xmm3, %xmm4 +; SSSE3-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; SSSE3-NEXT: movdqa %xmm0, %xmm5 +; SSSE3-NEXT: pshufb %xmm4, %xmm5 +; SSSE3-NEXT: psrlw $4, %xmm2 +; SSSE3-NEXT: pand %xmm3, %xmm2 +; SSSE3-NEXT: pshufb %xmm2, %xmm0 +; SSSE3-NEXT: paddb %xmm5, %xmm0 +; SSSE3-NEXT: movdqa %xmm0, %xmm2 +; SSSE3-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSSE3-NEXT: psadbw %xmm1, %xmm2 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSSE3-NEXT: psadbw %xmm1, %xmm0 +; SSSE3-NEXT: packuswb %xmm2, %xmm0 ; SSSE3-NEXT: retq ; ; SSE41-LABEL: testv4i32u: ; SSE41: # BB#0: -; SSE41-NEXT: pextrd $1, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: movd %xmm0, %ecx -; SSE41-NEXT: bsfl %ecx, %ecx -; SSE41-NEXT: movd %ecx, %xmm1 -; SSE41-NEXT: pinsrd $1, %eax, %xmm1 -; SSE41-NEXT: pextrd $2, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: pinsrd $2, %eax, %xmm1 -; SSE41-NEXT: pextrd $3, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: pinsrd $3, %eax, %xmm1 -; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pxor %xmm1, %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: psubd %xmm0, %xmm2 +; SSE41-NEXT: pand %xmm0, %xmm2 +; SSE41-NEXT: psubd {{.*}}(%rip), %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSE41-NEXT: movdqa %xmm2, %xmm4 +; SSE41-NEXT: pand %xmm3, %xmm4 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pshufb %xmm4, %xmm5 +; SSE41-NEXT: psrlw $4, %xmm2 +; SSE41-NEXT: pand %xmm3, %xmm2 +; SSE41-NEXT: pshufb %xmm2, %xmm0 +; SSE41-NEXT: paddb %xmm5, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm2 +; SSE41-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSE41-NEXT: psadbw %xmm1, %xmm2 +; SSE41-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE41-NEXT: psadbw %xmm1, %xmm0 +; SSE41-NEXT: packuswb %xmm2, %xmm0 ; SSE41-NEXT: retq ; -; AVX-LABEL: testv4i32u: -; AVX: # BB#0: -; AVX-NEXT: vpextrd $1, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vmovd %xmm0, %ecx -; AVX-NEXT: bsfl %ecx, %ecx -; AVX-NEXT: vmovd %ecx, %xmm1 -; AVX-NEXT: vpinsrd $1, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrd $2, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrd $3, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vpinsrd $3, %eax, %xmm1, %xmm0 -; AVX-NEXT: retq +; AVX1-LABEL: testv4i32u: +; AVX1: # BB#0: +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpsubd %xmm0, %xmm1, %xmm2 +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpsubd {{.*}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0 +; AVX1-NEXT: vpaddb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX1-NEXT: vpsadbw %xmm1, %xmm2, %xmm2 +; AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: testv4i32u: +; AVX2: # BB#0: +; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX2-NEXT: vpsubd %xmm0, %xmm1, %xmm2 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %xmm2 +; AVX2-NEXT: vpsubd %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpshufb %xmm0, %xmm4, %xmm0 +; AVX2-NEXT: vpaddb %xmm3, %xmm0, %xmm0 +; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX2-NEXT: vpsadbw %xmm1, %xmm2, %xmm2 +; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpackuswb %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: retq %out = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %in, i1 -1) ret <4 x i32> %out } -define <8 x i16> @testv8i16(<8 x i16> %in) { +define <8 x i16> @testv8i16(<8 x i16> %in) nounwind { ; SSE2-LABEL: testv8i16: ; SSE2: # BB#0: -; SSE2-NEXT: pextrw $7, %xmm0, %eax -; SSE2-NEXT: bsfw %ax, %cx -; SSE2-NEXT: movw $16, %ax -; SSE2-NEXT: cmovew %ax, %cx -; SSE2-NEXT: movd %ecx, %xmm1 -; SSE2-NEXT: pextrw $3, %xmm0, %ecx -; SSE2-NEXT: bsfw %cx, %cx -; SSE2-NEXT: cmovew %ax, %cx -; SSE2-NEXT: movd %ecx, %xmm2 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] -; SSE2-NEXT: pextrw $5, %xmm0, %ecx -; SSE2-NEXT: bsfw %cx, %cx -; SSE2-NEXT: cmovew %ax, %cx -; SSE2-NEXT: movd %ecx, %xmm3 -; SSE2-NEXT: pextrw $1, %xmm0, %ecx -; SSE2-NEXT: bsfw %cx, %cx -; SSE2-NEXT: cmovew %ax, %cx -; SSE2-NEXT: movd %ecx, %xmm1 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] -; SSE2-NEXT: pextrw $6, %xmm0, %ecx -; SSE2-NEXT: bsfw %cx, %cx -; SSE2-NEXT: cmovew %ax, %cx -; SSE2-NEXT: movd %ecx, %xmm2 -; SSE2-NEXT: pextrw $2, %xmm0, %ecx -; SSE2-NEXT: bsfw %cx, %cx -; SSE2-NEXT: cmovew %ax, %cx -; SSE2-NEXT: movd %ecx, %xmm3 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] -; SSE2-NEXT: pextrw $4, %xmm0, %ecx -; SSE2-NEXT: bsfw %cx, %cx -; SSE2-NEXT: cmovew %ax, %cx -; SSE2-NEXT: movd %ecx, %xmm2 -; SSE2-NEXT: movd %xmm0, %ecx -; SSE2-NEXT: bsfw %cx, %cx -; SSE2-NEXT: cmovew %ax, %cx -; SSE2-NEXT: movd %ecx, %xmm0 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: psubw %xmm0, %xmm1 +; SSE2-NEXT: pand %xmm0, %xmm1 +; SSE2-NEXT: psubw {{.*}}(%rip), %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: psrlw $1, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: psubw %xmm0, %xmm1 +; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [13107,13107,13107,13107,13107,13107,13107,13107] +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: pand %xmm0, %xmm2 +; SSE2-NEXT: psrlw $2, %xmm1 +; SSE2-NEXT: pand %xmm0, %xmm1 +; SSE2-NEXT: paddw %xmm2, %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: psrlw $4, %xmm2 +; SSE2-NEXT: paddw %xmm1, %xmm2 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: psllw $8, %xmm0 +; SSE2-NEXT: paddb %xmm2, %xmm0 +; SSE2-NEXT: psrlw $8, %xmm0 ; SSE2-NEXT: retq ; ; SSE3-LABEL: testv8i16: ; SSE3: # BB#0: -; SSE3-NEXT: pextrw $7, %xmm0, %eax -; SSE3-NEXT: bsfw %ax, %cx -; SSE3-NEXT: movw $16, %ax -; SSE3-NEXT: cmovew %ax, %cx -; SSE3-NEXT: movd %ecx, %xmm1 -; SSE3-NEXT: pextrw $3, %xmm0, %ecx -; SSE3-NEXT: bsfw %cx, %cx -; SSE3-NEXT: cmovew %ax, %cx -; SSE3-NEXT: movd %ecx, %xmm2 -; SSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] -; SSE3-NEXT: pextrw $5, %xmm0, %ecx -; SSE3-NEXT: bsfw %cx, %cx -; SSE3-NEXT: cmovew %ax, %cx -; SSE3-NEXT: movd %ecx, %xmm3 -; SSE3-NEXT: pextrw $1, %xmm0, %ecx -; SSE3-NEXT: bsfw %cx, %cx -; SSE3-NEXT: cmovew %ax, %cx -; SSE3-NEXT: movd %ecx, %xmm1 -; SSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3] -; SSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] -; SSE3-NEXT: pextrw $6, %xmm0, %ecx -; SSE3-NEXT: bsfw %cx, %cx -; SSE3-NEXT: cmovew %ax, %cx -; SSE3-NEXT: movd %ecx, %xmm2 -; SSE3-NEXT: pextrw $2, %xmm0, %ecx -; SSE3-NEXT: bsfw %cx, %cx -; SSE3-NEXT: cmovew %ax, %cx -; SSE3-NEXT: movd %ecx, %xmm3 -; SSE3-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] -; SSE3-NEXT: pextrw $4, %xmm0, %ecx -; SSE3-NEXT: bsfw %cx, %cx -; SSE3-NEXT: cmovew %ax, %cx -; SSE3-NEXT: movd %ecx, %xmm2 -; SSE3-NEXT: movd %xmm0, %ecx -; SSE3-NEXT: bsfw %cx, %cx -; SSE3-NEXT: cmovew %ax, %cx -; SSE3-NEXT: movd %ecx, %xmm0 -; SSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] -; SSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3] -; SSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE3-NEXT: pxor %xmm1, %xmm1 +; SSE3-NEXT: psubw %xmm0, %xmm1 +; SSE3-NEXT: pand %xmm0, %xmm1 +; SSE3-NEXT: psubw {{.*}}(%rip), %xmm1 +; SSE3-NEXT: movdqa %xmm1, %xmm0 +; SSE3-NEXT: psrlw $1, %xmm0 +; SSE3-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE3-NEXT: psubw %xmm0, %xmm1 +; SSE3-NEXT: movdqa {{.*#+}} xmm0 = [13107,13107,13107,13107,13107,13107,13107,13107] +; SSE3-NEXT: movdqa %xmm1, %xmm2 +; SSE3-NEXT: pand %xmm0, %xmm2 +; SSE3-NEXT: psrlw $2, %xmm1 +; SSE3-NEXT: pand %xmm0, %xmm1 +; SSE3-NEXT: paddw %xmm2, %xmm1 +; SSE3-NEXT: movdqa %xmm1, %xmm2 +; SSE3-NEXT: psrlw $4, %xmm2 +; SSE3-NEXT: paddw %xmm1, %xmm2 +; SSE3-NEXT: pand {{.*}}(%rip), %xmm2 +; SSE3-NEXT: movdqa %xmm2, %xmm0 +; SSE3-NEXT: psllw $8, %xmm0 +; SSE3-NEXT: paddb %xmm2, %xmm0 +; SSE3-NEXT: psrlw $8, %xmm0 ; SSE3-NEXT: retq ; ; SSSE3-LABEL: testv8i16: ; SSSE3: # BB#0: -; SSSE3-NEXT: pextrw $7, %xmm0, %eax -; SSSE3-NEXT: bsfw %ax, %cx -; SSSE3-NEXT: movw $16, %ax -; SSSE3-NEXT: cmovew %ax, %cx -; SSSE3-NEXT: movd %ecx, %xmm1 -; SSSE3-NEXT: pextrw $3, %xmm0, %ecx -; SSSE3-NEXT: bsfw %cx, %cx -; SSSE3-NEXT: cmovew %ax, %cx -; SSSE3-NEXT: movd %ecx, %xmm2 -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] -; SSSE3-NEXT: pextrw $5, %xmm0, %ecx -; SSSE3-NEXT: bsfw %cx, %cx -; SSSE3-NEXT: cmovew %ax, %cx -; SSSE3-NEXT: movd %ecx, %xmm3 -; SSSE3-NEXT: pextrw $1, %xmm0, %ecx -; SSSE3-NEXT: bsfw %cx, %cx -; SSSE3-NEXT: cmovew %ax, %cx -; SSSE3-NEXT: movd %ecx, %xmm1 -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3] -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] -; SSSE3-NEXT: pextrw $6, %xmm0, %ecx -; SSSE3-NEXT: bsfw %cx, %cx -; SSSE3-NEXT: cmovew %ax, %cx -; SSSE3-NEXT: movd %ecx, %xmm2 -; SSSE3-NEXT: pextrw $2, %xmm0, %ecx -; SSSE3-NEXT: bsfw %cx, %cx -; SSSE3-NEXT: cmovew %ax, %cx -; SSSE3-NEXT: movd %ecx, %xmm3 -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] -; SSSE3-NEXT: pextrw $4, %xmm0, %ecx -; SSSE3-NEXT: bsfw %cx, %cx -; SSSE3-NEXT: cmovew %ax, %cx -; SSSE3-NEXT: movd %ecx, %xmm2 -; SSSE3-NEXT: movd %xmm0, %ecx -; SSSE3-NEXT: bsfw %cx, %cx -; SSSE3-NEXT: cmovew %ax, %cx -; SSSE3-NEXT: movd %ecx, %xmm0 -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3] -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: psubw %xmm0, %xmm1 +; SSSE3-NEXT: pand %xmm0, %xmm1 +; SSSE3-NEXT: psubw {{.*}}(%rip), %xmm1 +; SSSE3-NEXT: movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSSE3-NEXT: movdqa %xmm1, %xmm2 +; SSSE3-NEXT: pand %xmm0, %xmm2 +; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; SSSE3-NEXT: movdqa %xmm3, %xmm4 +; SSSE3-NEXT: pshufb %xmm2, %xmm4 +; SSSE3-NEXT: psrlw $4, %xmm1 +; SSSE3-NEXT: pand %xmm0, %xmm1 +; SSSE3-NEXT: pshufb %xmm1, %xmm3 +; SSSE3-NEXT: paddb %xmm4, %xmm3 +; SSSE3-NEXT: movdqa %xmm3, %xmm0 +; SSSE3-NEXT: psllw $8, %xmm0 +; SSSE3-NEXT: paddb %xmm3, %xmm0 +; SSSE3-NEXT: psrlw $8, %xmm0 ; SSSE3-NEXT: retq ; ; SSE41-LABEL: testv8i16: ; SSE41: # BB#0: -; SSE41-NEXT: pextrw $1, %xmm0, %eax -; SSE41-NEXT: bsfw %ax, %cx -; SSE41-NEXT: movw $16, %ax -; SSE41-NEXT: cmovew %ax, %cx -; SSE41-NEXT: movd %xmm0, %edx -; SSE41-NEXT: bsfw %dx, %dx -; SSE41-NEXT: cmovew %ax, %dx -; SSE41-NEXT: movd %edx, %xmm1 -; SSE41-NEXT: pinsrw $1, %ecx, %xmm1 -; SSE41-NEXT: pextrw $2, %xmm0, %ecx -; SSE41-NEXT: bsfw %cx, %cx -; SSE41-NEXT: cmovew %ax, %cx -; SSE41-NEXT: pinsrw $2, %ecx, %xmm1 -; SSE41-NEXT: pextrw $3, %xmm0, %ecx -; SSE41-NEXT: bsfw %cx, %cx -; SSE41-NEXT: cmovew %ax, %cx -; SSE41-NEXT: pinsrw $3, %ecx, %xmm1 -; SSE41-NEXT: pextrw $4, %xmm0, %ecx -; SSE41-NEXT: bsfw %cx, %cx -; SSE41-NEXT: cmovew %ax, %cx -; SSE41-NEXT: pinsrw $4, %ecx, %xmm1 -; SSE41-NEXT: pextrw $5, %xmm0, %ecx -; SSE41-NEXT: bsfw %cx, %cx -; SSE41-NEXT: cmovew %ax, %cx -; SSE41-NEXT: pinsrw $5, %ecx, %xmm1 -; SSE41-NEXT: pextrw $6, %xmm0, %ecx -; SSE41-NEXT: bsfw %cx, %cx -; SSE41-NEXT: cmovew %ax, %cx -; SSE41-NEXT: pinsrw $6, %ecx, %xmm1 -; SSE41-NEXT: pextrw $7, %xmm0, %ecx -; SSE41-NEXT: bsfw %cx, %cx -; SSE41-NEXT: cmovew %ax, %cx -; SSE41-NEXT: pinsrw $7, %ecx, %xmm1 -; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pxor %xmm1, %xmm1 +; SSE41-NEXT: psubw %xmm0, %xmm1 +; SSE41-NEXT: pand %xmm0, %xmm1 +; SSE41-NEXT: psubw {{.*}}(%rip), %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSE41-NEXT: movdqa %xmm1, %xmm2 +; SSE41-NEXT: pand %xmm0, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: pshufb %xmm2, %xmm4 +; SSE41-NEXT: psrlw $4, %xmm1 +; SSE41-NEXT: pand %xmm0, %xmm1 +; SSE41-NEXT: pshufb %xmm1, %xmm3 +; SSE41-NEXT: paddb %xmm4, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm0 +; SSE41-NEXT: psllw $8, %xmm0 +; SSE41-NEXT: paddb %xmm3, %xmm0 +; SSE41-NEXT: psrlw $8, %xmm0 ; SSE41-NEXT: retq ; ; AVX-LABEL: testv8i16: ; AVX: # BB#0: -; AVX-NEXT: vpextrw $1, %xmm0, %eax -; AVX-NEXT: bsfw %ax, %cx -; AVX-NEXT: movw $16, %ax -; AVX-NEXT: cmovew %ax, %cx -; AVX-NEXT: vmovd %xmm0, %edx -; AVX-NEXT: bsfw %dx, %dx -; AVX-NEXT: cmovew %ax, %dx -; AVX-NEXT: vmovd %edx, %xmm1 -; AVX-NEXT: vpinsrw $1, %ecx, %xmm1, %xmm1 -; AVX-NEXT: vpextrw $2, %xmm0, %ecx -; AVX-NEXT: bsfw %cx, %cx -; AVX-NEXT: cmovew %ax, %cx -; AVX-NEXT: vpinsrw $2, %ecx, %xmm1, %xmm1 -; AVX-NEXT: vpextrw $3, %xmm0, %ecx -; AVX-NEXT: bsfw %cx, %cx -; AVX-NEXT: cmovew %ax, %cx -; AVX-NEXT: vpinsrw $3, %ecx, %xmm1, %xmm1 -; AVX-NEXT: vpextrw $4, %xmm0, %ecx -; AVX-NEXT: bsfw %cx, %cx -; AVX-NEXT: cmovew %ax, %cx -; AVX-NEXT: vpinsrw $4, %ecx, %xmm1, %xmm1 -; AVX-NEXT: vpextrw $5, %xmm0, %ecx -; AVX-NEXT: bsfw %cx, %cx -; AVX-NEXT: cmovew %ax, %cx -; AVX-NEXT: vpinsrw $5, %ecx, %xmm1, %xmm1 -; AVX-NEXT: vpextrw $6, %xmm0, %ecx -; AVX-NEXT: bsfw %cx, %cx -; AVX-NEXT: cmovew %ax, %cx -; AVX-NEXT: vpinsrw $6, %ecx, %xmm1, %xmm1 -; AVX-NEXT: vpextrw $7, %xmm0, %ecx -; AVX-NEXT: bsfw %cx, %cx -; AVX-NEXT: cmovew %ax, %cx -; AVX-NEXT: vpinsrw $7, %ecx, %xmm1, %xmm0 +; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpsubw %xmm0, %xmm1, %xmm1 +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpsubw {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm2 +; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm2 +; AVX-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpshufb %xmm0, %xmm3, %xmm0 +; AVX-NEXT: vpaddb %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpsllw $8, %xmm0, %xmm1 +; AVX-NEXT: vpaddb %xmm0, %xmm1, %xmm0 +; AVX-NEXT: vpsrlw $8, %xmm0, %xmm0 ; AVX-NEXT: retq %out = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %in, i1 0) ret <8 x i16> %out } -define <8 x i16> @testv8i16u(<8 x i16> %in) { +define <8 x i16> @testv8i16u(<8 x i16> %in) nounwind { ; SSE2-LABEL: testv8i16u: ; SSE2: # BB#0: -; SSE2-NEXT: pextrw $7, %xmm0, %eax -; SSE2-NEXT: bsfw %ax, %ax -; SSE2-NEXT: movd %eax, %xmm1 -; SSE2-NEXT: pextrw $3, %xmm0, %eax -; SSE2-NEXT: bsfw %ax, %ax -; SSE2-NEXT: movd %eax, %xmm2 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] -; SSE2-NEXT: pextrw $5, %xmm0, %eax -; SSE2-NEXT: bsfw %ax, %ax -; SSE2-NEXT: movd %eax, %xmm1 -; SSE2-NEXT: pextrw $1, %xmm0, %eax -; SSE2-NEXT: bsfw %ax, %ax -; SSE2-NEXT: movd %eax, %xmm3 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] -; SSE2-NEXT: pextrw $6, %xmm0, %eax -; SSE2-NEXT: bsfw %ax, %ax -; SSE2-NEXT: movd %eax, %xmm1 -; SSE2-NEXT: pextrw $2, %xmm0, %eax -; SSE2-NEXT: bsfw %ax, %ax -; SSE2-NEXT: movd %eax, %xmm2 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] -; SSE2-NEXT: pextrw $4, %xmm0, %eax -; SSE2-NEXT: bsfw %ax, %ax -; SSE2-NEXT: movd %eax, %xmm1 -; SSE2-NEXT: movd %xmm0, %eax -; SSE2-NEXT: bsfw %ax, %ax -; SSE2-NEXT: movd %eax, %xmm0 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3] +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: psubw %xmm0, %xmm1 +; SSE2-NEXT: pand %xmm0, %xmm1 +; SSE2-NEXT: psubw {{.*}}(%rip), %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: psrlw $1, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: psubw %xmm0, %xmm1 +; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [13107,13107,13107,13107,13107,13107,13107,13107] +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: pand %xmm0, %xmm2 +; SSE2-NEXT: psrlw $2, %xmm1 +; SSE2-NEXT: pand %xmm0, %xmm1 +; SSE2-NEXT: paddw %xmm2, %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: psrlw $4, %xmm2 +; SSE2-NEXT: paddw %xmm1, %xmm2 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: psllw $8, %xmm0 +; SSE2-NEXT: paddb %xmm2, %xmm0 +; SSE2-NEXT: psrlw $8, %xmm0 ; SSE2-NEXT: retq ; ; SSE3-LABEL: testv8i16u: ; SSE3: # BB#0: -; SSE3-NEXT: pextrw $7, %xmm0, %eax -; SSE3-NEXT: bsfw %ax, %ax -; SSE3-NEXT: movd %eax, %xmm1 -; SSE3-NEXT: pextrw $3, %xmm0, %eax -; SSE3-NEXT: bsfw %ax, %ax -; SSE3-NEXT: movd %eax, %xmm2 -; SSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] -; SSE3-NEXT: pextrw $5, %xmm0, %eax -; SSE3-NEXT: bsfw %ax, %ax -; SSE3-NEXT: movd %eax, %xmm1 -; SSE3-NEXT: pextrw $1, %xmm0, %eax -; SSE3-NEXT: bsfw %ax, %ax -; SSE3-NEXT: movd %eax, %xmm3 -; SSE3-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3] -; SSE3-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] -; SSE3-NEXT: pextrw $6, %xmm0, %eax -; SSE3-NEXT: bsfw %ax, %ax -; SSE3-NEXT: movd %eax, %xmm1 -; SSE3-NEXT: pextrw $2, %xmm0, %eax -; SSE3-NEXT: bsfw %ax, %ax -; SSE3-NEXT: movd %eax, %xmm2 -; SSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] -; SSE3-NEXT: pextrw $4, %xmm0, %eax -; SSE3-NEXT: bsfw %ax, %ax -; SSE3-NEXT: movd %eax, %xmm1 -; SSE3-NEXT: movd %xmm0, %eax -; SSE3-NEXT: bsfw %ax, %ax -; SSE3-NEXT: movd %eax, %xmm0 -; SSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] -; SSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] -; SSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3] +; SSE3-NEXT: pxor %xmm1, %xmm1 +; SSE3-NEXT: psubw %xmm0, %xmm1 +; SSE3-NEXT: pand %xmm0, %xmm1 +; SSE3-NEXT: psubw {{.*}}(%rip), %xmm1 +; SSE3-NEXT: movdqa %xmm1, %xmm0 +; SSE3-NEXT: psrlw $1, %xmm0 +; SSE3-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE3-NEXT: psubw %xmm0, %xmm1 +; SSE3-NEXT: movdqa {{.*#+}} xmm0 = [13107,13107,13107,13107,13107,13107,13107,13107] +; SSE3-NEXT: movdqa %xmm1, %xmm2 +; SSE3-NEXT: pand %xmm0, %xmm2 +; SSE3-NEXT: psrlw $2, %xmm1 +; SSE3-NEXT: pand %xmm0, %xmm1 +; SSE3-NEXT: paddw %xmm2, %xmm1 +; SSE3-NEXT: movdqa %xmm1, %xmm2 +; SSE3-NEXT: psrlw $4, %xmm2 +; SSE3-NEXT: paddw %xmm1, %xmm2 +; SSE3-NEXT: pand {{.*}}(%rip), %xmm2 +; SSE3-NEXT: movdqa %xmm2, %xmm0 +; SSE3-NEXT: psllw $8, %xmm0 +; SSE3-NEXT: paddb %xmm2, %xmm0 +; SSE3-NEXT: psrlw $8, %xmm0 ; SSE3-NEXT: retq ; ; SSSE3-LABEL: testv8i16u: ; SSSE3: # BB#0: -; SSSE3-NEXT: pextrw $7, %xmm0, %eax -; SSSE3-NEXT: bsfw %ax, %ax -; SSSE3-NEXT: movd %eax, %xmm1 -; SSSE3-NEXT: pextrw $3, %xmm0, %eax -; SSSE3-NEXT: bsfw %ax, %ax -; SSSE3-NEXT: movd %eax, %xmm2 -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] -; SSSE3-NEXT: pextrw $5, %xmm0, %eax -; SSSE3-NEXT: bsfw %ax, %ax -; SSSE3-NEXT: movd %eax, %xmm1 -; SSSE3-NEXT: pextrw $1, %xmm0, %eax -; SSSE3-NEXT: bsfw %ax, %ax -; SSSE3-NEXT: movd %eax, %xmm3 -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3] -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] -; SSSE3-NEXT: pextrw $6, %xmm0, %eax -; SSSE3-NEXT: bsfw %ax, %ax -; SSSE3-NEXT: movd %eax, %xmm1 -; SSSE3-NEXT: pextrw $2, %xmm0, %eax -; SSSE3-NEXT: bsfw %ax, %ax -; SSSE3-NEXT: movd %eax, %xmm2 -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] -; SSSE3-NEXT: pextrw $4, %xmm0, %eax -; SSSE3-NEXT: bsfw %ax, %ax -; SSSE3-NEXT: movd %eax, %xmm1 -; SSSE3-NEXT: movd %xmm0, %eax -; SSSE3-NEXT: bsfw %ax, %ax -; SSSE3-NEXT: movd %eax, %xmm0 -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3] +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: psubw %xmm0, %xmm1 +; SSSE3-NEXT: pand %xmm0, %xmm1 +; SSSE3-NEXT: psubw {{.*}}(%rip), %xmm1 +; SSSE3-NEXT: movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSSE3-NEXT: movdqa %xmm1, %xmm2 +; SSSE3-NEXT: pand %xmm0, %xmm2 +; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; SSSE3-NEXT: movdqa %xmm3, %xmm4 +; SSSE3-NEXT: pshufb %xmm2, %xmm4 +; SSSE3-NEXT: psrlw $4, %xmm1 +; SSSE3-NEXT: pand %xmm0, %xmm1 +; SSSE3-NEXT: pshufb %xmm1, %xmm3 +; SSSE3-NEXT: paddb %xmm4, %xmm3 +; SSSE3-NEXT: movdqa %xmm3, %xmm0 +; SSSE3-NEXT: psllw $8, %xmm0 +; SSSE3-NEXT: paddb %xmm3, %xmm0 +; SSSE3-NEXT: psrlw $8, %xmm0 ; SSSE3-NEXT: retq ; ; SSE41-LABEL: testv8i16u: ; SSE41: # BB#0: -; SSE41-NEXT: pextrw $1, %xmm0, %eax -; SSE41-NEXT: bsfw %ax, %ax -; SSE41-NEXT: movd %xmm0, %ecx -; SSE41-NEXT: bsfw %cx, %cx -; SSE41-NEXT: movd %ecx, %xmm1 -; SSE41-NEXT: pinsrw $1, %eax, %xmm1 -; SSE41-NEXT: pextrw $2, %xmm0, %eax -; SSE41-NEXT: bsfw %ax, %ax -; SSE41-NEXT: pinsrw $2, %eax, %xmm1 -; SSE41-NEXT: pextrw $3, %xmm0, %eax -; SSE41-NEXT: bsfw %ax, %ax -; SSE41-NEXT: pinsrw $3, %eax, %xmm1 -; SSE41-NEXT: pextrw $4, %xmm0, %eax -; SSE41-NEXT: bsfw %ax, %ax -; SSE41-NEXT: pinsrw $4, %eax, %xmm1 -; SSE41-NEXT: pextrw $5, %xmm0, %eax -; SSE41-NEXT: bsfw %ax, %ax -; SSE41-NEXT: pinsrw $5, %eax, %xmm1 -; SSE41-NEXT: pextrw $6, %xmm0, %eax -; SSE41-NEXT: bsfw %ax, %ax -; SSE41-NEXT: pinsrw $6, %eax, %xmm1 -; SSE41-NEXT: pextrw $7, %xmm0, %eax -; SSE41-NEXT: bsfw %ax, %ax -; SSE41-NEXT: pinsrw $7, %eax, %xmm1 -; SSE41-NEXT: movdqa %xmm1, %xmm0 -; SSE41-NEXT: retq +; SSE41-NEXT: pxor %xmm1, %xmm1 +; SSE41-NEXT: psubw %xmm0, %xmm1 +; SSE41-NEXT: pand %xmm0, %xmm1 +; SSE41-NEXT: psubw {{.*}}(%rip), %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSE41-NEXT: movdqa %xmm1, %xmm2 +; SSE41-NEXT: pand %xmm0, %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: pshufb %xmm2, %xmm4 +; SSE41-NEXT: psrlw $4, %xmm1 +; SSE41-NEXT: pand %xmm0, %xmm1 +; SSE41-NEXT: pshufb %xmm1, %xmm3 +; SSE41-NEXT: paddb %xmm4, %xmm3 +; SSE41-NEXT: movdqa %xmm3, %xmm0 +; SSE41-NEXT: psllw $8, %xmm0 +; SSE41-NEXT: paddb %xmm3, %xmm0 +; SSE41-NEXT: psrlw $8, %xmm0 +; SSE41-NEXT: retq ; ; AVX-LABEL: testv8i16u: ; AVX: # BB#0: -; AVX-NEXT: vpextrw $1, %xmm0, %eax -; AVX-NEXT: bsfw %ax, %ax -; AVX-NEXT: vmovd %xmm0, %ecx -; AVX-NEXT: bsfw %cx, %cx -; AVX-NEXT: vmovd %ecx, %xmm1 -; AVX-NEXT: vpinsrw $1, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrw $2, %xmm0, %eax -; AVX-NEXT: bsfw %ax, %ax -; AVX-NEXT: vpinsrw $2, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrw $3, %xmm0, %eax -; AVX-NEXT: bsfw %ax, %ax -; AVX-NEXT: vpinsrw $3, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrw $4, %xmm0, %eax -; AVX-NEXT: bsfw %ax, %ax -; AVX-NEXT: vpinsrw $4, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrw $5, %xmm0, %eax -; AVX-NEXT: bsfw %ax, %ax -; AVX-NEXT: vpinsrw $5, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrw $6, %xmm0, %eax -; AVX-NEXT: bsfw %ax, %ax -; AVX-NEXT: vpinsrw $6, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrw $7, %xmm0, %eax -; AVX-NEXT: bsfw %ax, %ax -; AVX-NEXT: vpinsrw $7, %eax, %xmm1, %xmm0 +; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpsubw %xmm0, %xmm1, %xmm1 +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpsubw {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm2 +; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm2 +; AVX-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpshufb %xmm0, %xmm3, %xmm0 +; AVX-NEXT: vpaddb %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpsllw $8, %xmm0, %xmm1 +; AVX-NEXT: vpaddb %xmm0, %xmm1, %xmm0 +; AVX-NEXT: vpsrlw $8, %xmm0, %xmm0 ; AVX-NEXT: retq %out = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %in, i1 -1) ret <8 x i16> %out } -define <16 x i8> @testv16i8(<16 x i8> %in) { +define <16 x i8> @testv16i8(<16 x i8> %in) nounwind { ; SSE2-LABEL: testv16i8: ; SSE2: # BB#0: -; SSE2: pushq %rbp -; SSE2: pushq %r14 -; SSE2: pushq %rbx -; SSE2: movaps %xmm0, -16(%rsp) -; SSE2-NEXT: movzbl -1(%rsp), %eax -; SSE2-NEXT: bsfl %eax, %edx -; SSE2-NEXT: movl $32, %eax -; SSE2-NEXT: cmovel %eax, %edx -; SSE2-NEXT: cmpl $32, %edx -; SSE2-NEXT: movl $8, %ecx -; SSE2-NEXT: cmovel %ecx, %edx -; SSE2-NEXT: movd %edx, %xmm0 -; SSE2-NEXT: movzbl -2(%rsp), %r14d -; SSE2-NEXT: movzbl -3(%rsp), %ebx -; SSE2-NEXT: movzbl -4(%rsp), %r9d -; SSE2-NEXT: movzbl -5(%rsp), %edi -; SSE2-NEXT: movzbl -6(%rsp), %r11d -; SSE2-NEXT: movzbl -7(%rsp), %edx -; SSE2-NEXT: movzbl -8(%rsp), %r8d -; SSE2-NEXT: movzbl -9(%rsp), %esi -; SSE2-NEXT: bsfl %esi, %esi -; SSE2-NEXT: cmovel %eax, %esi -; SSE2-NEXT: cmpl $32, %esi -; SSE2-NEXT: cmovel %ecx, %esi -; SSE2-NEXT: movd %esi, %xmm1 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] -; SSE2-NEXT: bsfl %edi, %esi -; SSE2-NEXT: cmovel %eax, %esi -; SSE2-NEXT: cmpl $32, %esi -; SSE2-NEXT: cmovel %ecx, %esi -; SSE2-NEXT: movd %esi, %xmm2 -; SSE2-NEXT: movzbl -10(%rsp), %edi -; SSE2-NEXT: movzbl -11(%rsp), %esi -; SSE2-NEXT: movzbl -12(%rsp), %r10d -; SSE2-NEXT: movzbl -13(%rsp), %ebp -; SSE2-NEXT: bsfl %ebp, %ebp -; SSE2-NEXT: cmovel %eax, %ebp -; SSE2-NEXT: cmpl $32, %ebp -; SSE2-NEXT: cmovel %ecx, %ebp -; SSE2-NEXT: movd %ebp, %xmm0 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] -; SSE2-NEXT: bsfl %ebx, %ebx -; SSE2-NEXT: cmovel %eax, %ebx -; SSE2-NEXT: cmpl $32, %ebx -; SSE2-NEXT: cmovel %ecx, %ebx -; SSE2-NEXT: movd %ebx, %xmm1 -; SSE2-NEXT: bsfl %esi, %esi -; SSE2-NEXT: cmovel %eax, %esi -; SSE2-NEXT: cmpl $32, %esi -; SSE2-NEXT: cmovel %ecx, %esi -; SSE2-NEXT: movd %esi, %xmm2 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] -; SSE2-NEXT: bsfl %edx, %edx -; SSE2-NEXT: cmovel %eax, %edx -; SSE2-NEXT: cmpl $32, %edx -; SSE2-NEXT: cmovel %ecx, %edx -; SSE2-NEXT: movd %edx, %xmm3 -; SSE2-NEXT: movzbl -14(%rsp), %edx -; SSE2-NEXT: movzbl -15(%rsp), %esi -; SSE2-NEXT: bsfl %esi, %esi -; SSE2-NEXT: cmovel %eax, %esi -; SSE2-NEXT: cmpl $32, %esi -; SSE2-NEXT: cmovel %ecx, %esi -; SSE2-NEXT: movd %esi, %xmm1 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7] -; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] -; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] -; SSE2-NEXT: bsfl %r14d, %esi -; SSE2-NEXT: cmovel %eax, %esi -; SSE2-NEXT: cmpl $32, %esi -; SSE2-NEXT: cmovel %ecx, %esi -; SSE2-NEXT: movd %esi, %xmm0 -; SSE2-NEXT: bsfl %edi, %esi -; SSE2-NEXT: cmovel %eax, %esi -; SSE2-NEXT: cmpl $32, %esi -; SSE2-NEXT: cmovel %ecx, %esi -; SSE2-NEXT: movd %esi, %xmm3 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] -; SSE2-NEXT: bsfl %r11d, %esi -; SSE2-NEXT: cmovel %eax, %esi -; SSE2-NEXT: cmpl $32, %esi -; SSE2-NEXT: cmovel %ecx, %esi -; SSE2-NEXT: movd %esi, %xmm0 -; SSE2-NEXT: bsfl %edx, %edx -; SSE2-NEXT: cmovel %eax, %edx -; SSE2-NEXT: cmpl $32, %edx -; SSE2-NEXT: cmovel %ecx, %edx -; SSE2-NEXT: movd %edx, %xmm2 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] -; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] -; SSE2-NEXT: bsfl %r9d, %edx -; SSE2-NEXT: cmovel %eax, %edx -; SSE2-NEXT: cmpl $32, %edx -; SSE2-NEXT: cmovel %ecx, %edx -; SSE2-NEXT: movd %edx, %xmm0 -; SSE2-NEXT: bsfl %r10d, %edx -; SSE2-NEXT: cmovel %eax, %edx -; SSE2-NEXT: cmpl $32, %edx -; SSE2-NEXT: cmovel %ecx, %edx -; SSE2-NEXT: movd %edx, %xmm3 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] -; SSE2-NEXT: bsfl %r8d, %edx -; SSE2-NEXT: cmovel %eax, %edx -; SSE2-NEXT: cmpl $32, %edx -; SSE2-NEXT: cmovel %ecx, %edx -; SSE2-NEXT: movd %edx, %xmm4 -; SSE2-NEXT: movzbl -16(%rsp), %edx -; SSE2-NEXT: bsfl %edx, %edx -; SSE2-NEXT: cmovel %eax, %edx -; SSE2-NEXT: cmpl $32, %edx -; SSE2-NEXT: cmovel %ecx, %edx -; SSE2-NEXT: movd %edx, %xmm0 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7] -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] -; SSE2-NEXT: popq %rbx -; SSE2-NEXT: popq %r14 -; SSE2-NEXT: popq %rbp +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: psubb %xmm0, %xmm1 +; SSE2-NEXT: pand %xmm0, %xmm1 +; SSE2-NEXT: psubb {{.*}}(%rip), %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: psrlw $1, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: psubb %xmm0, %xmm1 +; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51] +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: pand %xmm0, %xmm2 +; SSE2-NEXT: psrlw $2, %xmm1 +; SSE2-NEXT: pand %xmm0, %xmm1 +; SSE2-NEXT: paddb %xmm2, %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: psrlw $4, %xmm0 +; SSE2-NEXT: paddb %xmm1, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 ; SSE2-NEXT: retq ; ; SSE3-LABEL: testv16i8: ; SSE3: # BB#0: -; SSE3: pushq %rbp -; SSE3: pushq %r14 -; SSE3: pushq %rbx -; SSE3: movaps %xmm0, -16(%rsp) -; SSE3-NEXT: movzbl -1(%rsp), %eax -; SSE3-NEXT: bsfl %eax, %edx -; SSE3-NEXT: movl $32, %eax -; SSE3-NEXT: cmovel %eax, %edx -; SSE3-NEXT: cmpl $32, %edx -; SSE3-NEXT: movl $8, %ecx -; SSE3-NEXT: cmovel %ecx, %edx -; SSE3-NEXT: movd %edx, %xmm0 -; SSE3-NEXT: movzbl -2(%rsp), %r14d -; SSE3-NEXT: movzbl -3(%rsp), %ebx -; SSE3-NEXT: movzbl -4(%rsp), %r9d -; SSE3-NEXT: movzbl -5(%rsp), %edi -; SSE3-NEXT: movzbl -6(%rsp), %r11d -; SSE3-NEXT: movzbl -7(%rsp), %edx -; SSE3-NEXT: movzbl -8(%rsp), %r8d -; SSE3-NEXT: movzbl -9(%rsp), %esi -; SSE3-NEXT: bsfl %esi, %esi -; SSE3-NEXT: cmovel %eax, %esi -; SSE3-NEXT: cmpl $32, %esi -; SSE3-NEXT: cmovel %ecx, %esi -; SSE3-NEXT: movd %esi, %xmm1 -; SSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] -; SSE3-NEXT: bsfl %edi, %esi -; SSE3-NEXT: cmovel %eax, %esi -; SSE3-NEXT: cmpl $32, %esi -; SSE3-NEXT: cmovel %ecx, %esi -; SSE3-NEXT: movd %esi, %xmm2 -; SSE3-NEXT: movzbl -10(%rsp), %edi -; SSE3-NEXT: movzbl -11(%rsp), %esi -; SSE3-NEXT: movzbl -12(%rsp), %r10d -; SSE3-NEXT: movzbl -13(%rsp), %ebp -; SSE3-NEXT: bsfl %ebp, %ebp -; SSE3-NEXT: cmovel %eax, %ebp -; SSE3-NEXT: cmpl $32, %ebp -; SSE3-NEXT: cmovel %ecx, %ebp -; SSE3-NEXT: movd %ebp, %xmm0 -; SSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; SSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] -; SSE3-NEXT: bsfl %ebx, %ebx -; SSE3-NEXT: cmovel %eax, %ebx -; SSE3-NEXT: cmpl $32, %ebx -; SSE3-NEXT: cmovel %ecx, %ebx -; SSE3-NEXT: movd %ebx, %xmm1 -; SSE3-NEXT: bsfl %esi, %esi -; SSE3-NEXT: cmovel %eax, %esi -; SSE3-NEXT: cmpl $32, %esi -; SSE3-NEXT: cmovel %ecx, %esi -; SSE3-NEXT: movd %esi, %xmm2 -; SSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] -; SSE3-NEXT: bsfl %edx, %edx -; SSE3-NEXT: cmovel %eax, %edx -; SSE3-NEXT: cmpl $32, %edx -; SSE3-NEXT: cmovel %ecx, %edx -; SSE3-NEXT: movd %edx, %xmm3 -; SSE3-NEXT: movzbl -14(%rsp), %edx -; SSE3-NEXT: movzbl -15(%rsp), %esi -; SSE3-NEXT: bsfl %esi, %esi -; SSE3-NEXT: cmovel %eax, %esi -; SSE3-NEXT: cmpl $32, %esi -; SSE3-NEXT: cmovel %ecx, %esi -; SSE3-NEXT: movd %esi, %xmm1 -; SSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7] -; SSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] -; SSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] -; SSE3-NEXT: bsfl %r14d, %esi -; SSE3-NEXT: cmovel %eax, %esi -; SSE3-NEXT: cmpl $32, %esi -; SSE3-NEXT: cmovel %ecx, %esi -; SSE3-NEXT: movd %esi, %xmm0 -; SSE3-NEXT: bsfl %edi, %esi -; SSE3-NEXT: cmovel %eax, %esi -; SSE3-NEXT: cmpl $32, %esi -; SSE3-NEXT: cmovel %ecx, %esi -; SSE3-NEXT: movd %esi, %xmm3 -; SSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] -; SSE3-NEXT: bsfl %r11d, %esi -; SSE3-NEXT: cmovel %eax, %esi -; SSE3-NEXT: cmpl $32, %esi -; SSE3-NEXT: cmovel %ecx, %esi -; SSE3-NEXT: movd %esi, %xmm0 -; SSE3-NEXT: bsfl %edx, %edx -; SSE3-NEXT: cmovel %eax, %edx -; SSE3-NEXT: cmpl $32, %edx -; SSE3-NEXT: cmovel %ecx, %edx -; SSE3-NEXT: movd %edx, %xmm2 -; SSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] -; SSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] -; SSE3-NEXT: bsfl %r9d, %edx -; SSE3-NEXT: cmovel %eax, %edx -; SSE3-NEXT: cmpl $32, %edx -; SSE3-NEXT: cmovel %ecx, %edx -; SSE3-NEXT: movd %edx, %xmm0 -; SSE3-NEXT: bsfl %r10d, %edx -; SSE3-NEXT: cmovel %eax, %edx -; SSE3-NEXT: cmpl $32, %edx -; SSE3-NEXT: cmovel %ecx, %edx -; SSE3-NEXT: movd %edx, %xmm3 -; SSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] -; SSE3-NEXT: bsfl %r8d, %edx -; SSE3-NEXT: cmovel %eax, %edx -; SSE3-NEXT: cmpl $32, %edx -; SSE3-NEXT: cmovel %ecx, %edx -; SSE3-NEXT: movd %edx, %xmm4 -; SSE3-NEXT: movzbl -16(%rsp), %edx -; SSE3-NEXT: bsfl %edx, %edx -; SSE3-NEXT: cmovel %eax, %edx -; SSE3-NEXT: cmpl $32, %edx -; SSE3-NEXT: cmovel %ecx, %edx -; SSE3-NEXT: movd %edx, %xmm0 -; SSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7] -; SSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] -; SSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; SSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] -; SSE3-NEXT: popq %rbx -; SSE3-NEXT: popq %r14 -; SSE3-NEXT: popq %rbp +; SSE3-NEXT: pxor %xmm1, %xmm1 +; SSE3-NEXT: psubb %xmm0, %xmm1 +; SSE3-NEXT: pand %xmm0, %xmm1 +; SSE3-NEXT: psubb {{.*}}(%rip), %xmm1 +; SSE3-NEXT: movdqa %xmm1, %xmm0 +; SSE3-NEXT: psrlw $1, %xmm0 +; SSE3-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE3-NEXT: psubb %xmm0, %xmm1 +; SSE3-NEXT: movdqa {{.*#+}} xmm0 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51] +; SSE3-NEXT: movdqa %xmm1, %xmm2 +; SSE3-NEXT: pand %xmm0, %xmm2 +; SSE3-NEXT: psrlw $2, %xmm1 +; SSE3-NEXT: pand %xmm0, %xmm1 +; SSE3-NEXT: paddb %xmm2, %xmm1 +; SSE3-NEXT: movdqa %xmm1, %xmm0 +; SSE3-NEXT: psrlw $4, %xmm0 +; SSE3-NEXT: paddb %xmm1, %xmm0 +; SSE3-NEXT: pand {{.*}}(%rip), %xmm0 ; SSE3-NEXT: retq ; ; SSSE3-LABEL: testv16i8: ; SSSE3: # BB#0: -; SSSE3: pushq %rbp -; SSSE3: pushq %r14 -; SSSE3: pushq %rbx -; SSSE3: movaps %xmm0, -16(%rsp) -; SSSE3-NEXT: movzbl -1(%rsp), %eax -; SSSE3-NEXT: bsfl %eax, %edx -; SSSE3-NEXT: movl $32, %eax -; SSSE3-NEXT: cmovel %eax, %edx -; SSSE3-NEXT: cmpl $32, %edx -; SSSE3-NEXT: movl $8, %ecx -; SSSE3-NEXT: cmovel %ecx, %edx -; SSSE3-NEXT: movd %edx, %xmm0 -; SSSE3-NEXT: movzbl -2(%rsp), %r14d -; SSSE3-NEXT: movzbl -3(%rsp), %ebx -; SSSE3-NEXT: movzbl -4(%rsp), %r9d -; SSSE3-NEXT: movzbl -5(%rsp), %edi -; SSSE3-NEXT: movzbl -6(%rsp), %r11d -; SSSE3-NEXT: movzbl -7(%rsp), %edx -; SSSE3-NEXT: movzbl -8(%rsp), %r8d -; SSSE3-NEXT: movzbl -9(%rsp), %esi -; SSSE3-NEXT: bsfl %esi, %esi -; SSSE3-NEXT: cmovel %eax, %esi -; SSSE3-NEXT: cmpl $32, %esi -; SSSE3-NEXT: cmovel %ecx, %esi -; SSSE3-NEXT: movd %esi, %xmm1 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] -; SSSE3-NEXT: bsfl %edi, %esi -; SSSE3-NEXT: cmovel %eax, %esi -; SSSE3-NEXT: cmpl $32, %esi -; SSSE3-NEXT: cmovel %ecx, %esi -; SSSE3-NEXT: movd %esi, %xmm2 -; SSSE3-NEXT: movzbl -10(%rsp), %edi -; SSSE3-NEXT: movzbl -11(%rsp), %esi -; SSSE3-NEXT: movzbl -12(%rsp), %r10d -; SSSE3-NEXT: movzbl -13(%rsp), %ebp -; SSSE3-NEXT: bsfl %ebp, %ebp -; SSSE3-NEXT: cmovel %eax, %ebp -; SSSE3-NEXT: cmpl $32, %ebp -; SSSE3-NEXT: cmovel %ecx, %ebp -; SSSE3-NEXT: movd %ebp, %xmm0 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] -; SSSE3-NEXT: bsfl %ebx, %ebx -; SSSE3-NEXT: cmovel %eax, %ebx -; SSSE3-NEXT: cmpl $32, %ebx -; SSSE3-NEXT: cmovel %ecx, %ebx -; SSSE3-NEXT: movd %ebx, %xmm1 -; SSSE3-NEXT: bsfl %esi, %esi -; SSSE3-NEXT: cmovel %eax, %esi -; SSSE3-NEXT: cmpl $32, %esi -; SSSE3-NEXT: cmovel %ecx, %esi -; SSSE3-NEXT: movd %esi, %xmm2 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] -; SSSE3-NEXT: bsfl %edx, %edx -; SSSE3-NEXT: cmovel %eax, %edx -; SSSE3-NEXT: cmpl $32, %edx -; SSSE3-NEXT: cmovel %ecx, %edx -; SSSE3-NEXT: movd %edx, %xmm3 -; SSSE3-NEXT: movzbl -14(%rsp), %edx -; SSSE3-NEXT: movzbl -15(%rsp), %esi -; SSSE3-NEXT: bsfl %esi, %esi -; SSSE3-NEXT: cmovel %eax, %esi -; SSSE3-NEXT: cmpl $32, %esi -; SSSE3-NEXT: cmovel %ecx, %esi -; SSSE3-NEXT: movd %esi, %xmm1 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7] -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] -; SSSE3-NEXT: bsfl %r14d, %esi -; SSSE3-NEXT: cmovel %eax, %esi -; SSSE3-NEXT: cmpl $32, %esi -; SSSE3-NEXT: cmovel %ecx, %esi -; SSSE3-NEXT: movd %esi, %xmm0 -; SSSE3-NEXT: bsfl %edi, %esi -; SSSE3-NEXT: cmovel %eax, %esi -; SSSE3-NEXT: cmpl $32, %esi -; SSSE3-NEXT: cmovel %ecx, %esi -; SSSE3-NEXT: movd %esi, %xmm3 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] -; SSSE3-NEXT: bsfl %r11d, %esi -; SSSE3-NEXT: cmovel %eax, %esi -; SSSE3-NEXT: cmpl $32, %esi -; SSSE3-NEXT: cmovel %ecx, %esi -; SSSE3-NEXT: movd %esi, %xmm0 -; SSSE3-NEXT: bsfl %edx, %edx -; SSSE3-NEXT: cmovel %eax, %edx -; SSSE3-NEXT: cmpl $32, %edx -; SSSE3-NEXT: cmovel %ecx, %edx -; SSSE3-NEXT: movd %edx, %xmm2 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] -; SSSE3-NEXT: bsfl %r9d, %edx -; SSSE3-NEXT: cmovel %eax, %edx -; SSSE3-NEXT: cmpl $32, %edx -; SSSE3-NEXT: cmovel %ecx, %edx -; SSSE3-NEXT: movd %edx, %xmm0 -; SSSE3-NEXT: bsfl %r10d, %edx -; SSSE3-NEXT: cmovel %eax, %edx -; SSSE3-NEXT: cmpl $32, %edx -; SSSE3-NEXT: cmovel %ecx, %edx -; SSSE3-NEXT: movd %edx, %xmm3 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] -; SSSE3-NEXT: bsfl %r8d, %edx -; SSSE3-NEXT: cmovel %eax, %edx -; SSSE3-NEXT: cmpl $32, %edx -; SSSE3-NEXT: cmovel %ecx, %edx -; SSSE3-NEXT: movd %edx, %xmm4 -; SSSE3-NEXT: movzbl -16(%rsp), %edx -; SSSE3-NEXT: bsfl %edx, %edx -; SSSE3-NEXT: cmovel %eax, %edx -; SSSE3-NEXT: cmpl $32, %edx -; SSSE3-NEXT: cmovel %ecx, %edx -; SSSE3-NEXT: movd %edx, %xmm0 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7] -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] -; SSSE3-NEXT: popq %rbx -; SSSE3-NEXT: popq %r14 -; SSSE3-NEXT: popq %rbp +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: psubb %xmm0, %xmm1 +; SSSE3-NEXT: pand %xmm0, %xmm1 +; SSSE3-NEXT: psubb {{.*}}(%rip), %xmm1 +; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSSE3-NEXT: movdqa %xmm1, %xmm3 +; SSSE3-NEXT: pand %xmm2, %xmm3 +; SSSE3-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; SSSE3-NEXT: movdqa %xmm0, %xmm4 +; SSSE3-NEXT: pshufb %xmm3, %xmm4 +; SSSE3-NEXT: psrlw $4, %xmm1 +; SSSE3-NEXT: pand %xmm2, %xmm1 +; SSSE3-NEXT: pshufb %xmm1, %xmm0 +; SSSE3-NEXT: paddb %xmm4, %xmm0 ; SSSE3-NEXT: retq ; ; SSE41-LABEL: testv16i8: ; SSE41: # BB#0: -; SSE41-NEXT: pextrb $1, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %edx -; SSE41-NEXT: movl $32, %eax -; SSE41-NEXT: cmovel %eax, %edx -; SSE41-NEXT: cmpl $32, %edx -; SSE41-NEXT: movl $8, %ecx -; SSE41-NEXT: cmovel %ecx, %edx -; SSE41-NEXT: pextrb $0, %xmm0, %esi -; SSE41-NEXT: bsfl %esi, %esi -; SSE41-NEXT: cmovel %eax, %esi -; SSE41-NEXT: cmpl $32, %esi -; SSE41-NEXT: cmovel %ecx, %esi -; SSE41-NEXT: movd %esi, %xmm1 -; SSE41-NEXT: pinsrb $1, %edx, %xmm1 -; SSE41-NEXT: pextrb $2, %xmm0, %edx -; SSE41-NEXT: bsfl %edx, %edx -; SSE41-NEXT: cmovel %eax, %edx -; SSE41-NEXT: cmpl $32, %edx -; SSE41-NEXT: cmovel %ecx, %edx -; SSE41-NEXT: pinsrb $2, %edx, %xmm1 -; SSE41-NEXT: pextrb $3, %xmm0, %edx -; SSE41-NEXT: bsfl %edx, %edx -; SSE41-NEXT: cmovel %eax, %edx -; SSE41-NEXT: cmpl $32, %edx -; SSE41-NEXT: cmovel %ecx, %edx -; SSE41-NEXT: pinsrb $3, %edx, %xmm1 -; SSE41-NEXT: pextrb $4, %xmm0, %edx -; SSE41-NEXT: bsfl %edx, %edx -; SSE41-NEXT: cmovel %eax, %edx -; SSE41-NEXT: cmpl $32, %edx -; SSE41-NEXT: cmovel %ecx, %edx -; SSE41-NEXT: pinsrb $4, %edx, %xmm1 -; SSE41-NEXT: pextrb $5, %xmm0, %edx -; SSE41-NEXT: bsfl %edx, %edx -; SSE41-NEXT: cmovel %eax, %edx -; SSE41-NEXT: cmpl $32, %edx -; SSE41-NEXT: cmovel %ecx, %edx -; SSE41-NEXT: pinsrb $5, %edx, %xmm1 -; SSE41-NEXT: pextrb $6, %xmm0, %edx -; SSE41-NEXT: bsfl %edx, %edx -; SSE41-NEXT: cmovel %eax, %edx -; SSE41-NEXT: cmpl $32, %edx -; SSE41-NEXT: cmovel %ecx, %edx -; SSE41-NEXT: pinsrb $6, %edx, %xmm1 -; SSE41-NEXT: pextrb $7, %xmm0, %edx -; SSE41-NEXT: bsfl %edx, %edx -; SSE41-NEXT: cmovel %eax, %edx -; SSE41-NEXT: cmpl $32, %edx -; SSE41-NEXT: cmovel %ecx, %edx -; SSE41-NEXT: pinsrb $7, %edx, %xmm1 -; SSE41-NEXT: pextrb $8, %xmm0, %edx -; SSE41-NEXT: bsfl %edx, %edx -; SSE41-NEXT: cmovel %eax, %edx -; SSE41-NEXT: cmpl $32, %edx -; SSE41-NEXT: cmovel %ecx, %edx -; SSE41-NEXT: pinsrb $8, %edx, %xmm1 -; SSE41-NEXT: pextrb $9, %xmm0, %edx -; SSE41-NEXT: bsfl %edx, %edx -; SSE41-NEXT: cmovel %eax, %edx -; SSE41-NEXT: cmpl $32, %edx -; SSE41-NEXT: cmovel %ecx, %edx -; SSE41-NEXT: pinsrb $9, %edx, %xmm1 -; SSE41-NEXT: pextrb $10, %xmm0, %edx -; SSE41-NEXT: bsfl %edx, %edx -; SSE41-NEXT: cmovel %eax, %edx -; SSE41-NEXT: cmpl $32, %edx -; SSE41-NEXT: cmovel %ecx, %edx -; SSE41-NEXT: pinsrb $10, %edx, %xmm1 -; SSE41-NEXT: pextrb $11, %xmm0, %edx -; SSE41-NEXT: bsfl %edx, %edx -; SSE41-NEXT: cmovel %eax, %edx -; SSE41-NEXT: cmpl $32, %edx -; SSE41-NEXT: cmovel %ecx, %edx -; SSE41-NEXT: pinsrb $11, %edx, %xmm1 -; SSE41-NEXT: pextrb $12, %xmm0, %edx -; SSE41-NEXT: bsfl %edx, %edx -; SSE41-NEXT: cmovel %eax, %edx -; SSE41-NEXT: cmpl $32, %edx -; SSE41-NEXT: cmovel %ecx, %edx -; SSE41-NEXT: pinsrb $12, %edx, %xmm1 -; SSE41-NEXT: pextrb $13, %xmm0, %edx -; SSE41-NEXT: bsfl %edx, %edx -; SSE41-NEXT: cmovel %eax, %edx -; SSE41-NEXT: cmpl $32, %edx -; SSE41-NEXT: cmovel %ecx, %edx -; SSE41-NEXT: pinsrb $13, %edx, %xmm1 -; SSE41-NEXT: pextrb $14, %xmm0, %edx -; SSE41-NEXT: bsfl %edx, %edx -; SSE41-NEXT: cmovel %eax, %edx -; SSE41-NEXT: cmpl $32, %edx -; SSE41-NEXT: cmovel %ecx, %edx -; SSE41-NEXT: pinsrb $14, %edx, %xmm1 -; SSE41-NEXT: pextrb $15, %xmm0, %edx -; SSE41-NEXT: bsfl %edx, %edx -; SSE41-NEXT: cmovel %eax, %edx -; SSE41-NEXT: cmpl $32, %edx -; SSE41-NEXT: cmovel %ecx, %edx -; SSE41-NEXT: pinsrb $15, %edx, %xmm1 -; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pxor %xmm1, %xmm1 +; SSE41-NEXT: psubb %xmm0, %xmm1 +; SSE41-NEXT: pand %xmm0, %xmm1 +; SSE41-NEXT: psubb {{.*}}(%rip), %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSE41-NEXT: movdqa %xmm1, %xmm3 +; SSE41-NEXT: pand %xmm2, %xmm3 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; SSE41-NEXT: movdqa %xmm0, %xmm4 +; SSE41-NEXT: pshufb %xmm3, %xmm4 +; SSE41-NEXT: psrlw $4, %xmm1 +; SSE41-NEXT: pand %xmm2, %xmm1 +; SSE41-NEXT: pshufb %xmm1, %xmm0 +; SSE41-NEXT: paddb %xmm4, %xmm0 ; SSE41-NEXT: retq ; ; AVX-LABEL: testv16i8: ; AVX: # BB#0: -; AVX-NEXT: vpextrb $1, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %edx -; AVX-NEXT: movl $32, %eax -; AVX-NEXT: cmovel %eax, %edx -; AVX-NEXT: cmpl $32, %edx -; AVX-NEXT: movl $8, %ecx -; AVX-NEXT: cmovel %ecx, %edx -; AVX-NEXT: vpextrb $0, %xmm0, %esi -; AVX-NEXT: bsfl %esi, %esi -; AVX-NEXT: cmovel %eax, %esi -; AVX-NEXT: cmpl $32, %esi -; AVX-NEXT: cmovel %ecx, %esi -; AVX-NEXT: vmovd %esi, %xmm1 -; AVX-NEXT: vpinsrb $1, %edx, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $2, %xmm0, %edx -; AVX-NEXT: bsfl %edx, %edx -; AVX-NEXT: cmovel %eax, %edx -; AVX-NEXT: cmpl $32, %edx -; AVX-NEXT: cmovel %ecx, %edx -; AVX-NEXT: vpinsrb $2, %edx, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $3, %xmm0, %edx -; AVX-NEXT: bsfl %edx, %edx -; AVX-NEXT: cmovel %eax, %edx -; AVX-NEXT: cmpl $32, %edx -; AVX-NEXT: cmovel %ecx, %edx -; AVX-NEXT: vpinsrb $3, %edx, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $4, %xmm0, %edx -; AVX-NEXT: bsfl %edx, %edx -; AVX-NEXT: cmovel %eax, %edx -; AVX-NEXT: cmpl $32, %edx -; AVX-NEXT: cmovel %ecx, %edx -; AVX-NEXT: vpinsrb $4, %edx, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $5, %xmm0, %edx -; AVX-NEXT: bsfl %edx, %edx -; AVX-NEXT: cmovel %eax, %edx -; AVX-NEXT: cmpl $32, %edx -; AVX-NEXT: cmovel %ecx, %edx -; AVX-NEXT: vpinsrb $5, %edx, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $6, %xmm0, %edx -; AVX-NEXT: bsfl %edx, %edx -; AVX-NEXT: cmovel %eax, %edx -; AVX-NEXT: cmpl $32, %edx -; AVX-NEXT: cmovel %ecx, %edx -; AVX-NEXT: vpinsrb $6, %edx, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $7, %xmm0, %edx -; AVX-NEXT: bsfl %edx, %edx -; AVX-NEXT: cmovel %eax, %edx -; AVX-NEXT: cmpl $32, %edx -; AVX-NEXT: cmovel %ecx, %edx -; AVX-NEXT: vpinsrb $7, %edx, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $8, %xmm0, %edx -; AVX-NEXT: bsfl %edx, %edx -; AVX-NEXT: cmovel %eax, %edx -; AVX-NEXT: cmpl $32, %edx -; AVX-NEXT: cmovel %ecx, %edx -; AVX-NEXT: vpinsrb $8, %edx, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $9, %xmm0, %edx -; AVX-NEXT: bsfl %edx, %edx -; AVX-NEXT: cmovel %eax, %edx -; AVX-NEXT: cmpl $32, %edx -; AVX-NEXT: cmovel %ecx, %edx -; AVX-NEXT: vpinsrb $9, %edx, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $10, %xmm0, %edx -; AVX-NEXT: bsfl %edx, %edx -; AVX-NEXT: cmovel %eax, %edx -; AVX-NEXT: cmpl $32, %edx -; AVX-NEXT: cmovel %ecx, %edx -; AVX-NEXT: vpinsrb $10, %edx, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $11, %xmm0, %edx -; AVX-NEXT: bsfl %edx, %edx -; AVX-NEXT: cmovel %eax, %edx -; AVX-NEXT: cmpl $32, %edx -; AVX-NEXT: cmovel %ecx, %edx -; AVX-NEXT: vpinsrb $11, %edx, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $12, %xmm0, %edx -; AVX-NEXT: bsfl %edx, %edx -; AVX-NEXT: cmovel %eax, %edx -; AVX-NEXT: cmpl $32, %edx -; AVX-NEXT: cmovel %ecx, %edx -; AVX-NEXT: vpinsrb $12, %edx, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $13, %xmm0, %edx -; AVX-NEXT: bsfl %edx, %edx -; AVX-NEXT: cmovel %eax, %edx -; AVX-NEXT: cmpl $32, %edx -; AVX-NEXT: cmovel %ecx, %edx -; AVX-NEXT: vpinsrb $13, %edx, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $14, %xmm0, %edx -; AVX-NEXT: bsfl %edx, %edx -; AVX-NEXT: cmovel %eax, %edx -; AVX-NEXT: cmpl $32, %edx -; AVX-NEXT: cmovel %ecx, %edx -; AVX-NEXT: vpinsrb $14, %edx, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $15, %xmm0, %edx -; AVX-NEXT: bsfl %edx, %edx -; AVX-NEXT: cmovel %eax, %edx -; AVX-NEXT: cmpl $32, %edx -; AVX-NEXT: cmovel %ecx, %edx -; AVX-NEXT: vpinsrb $15, %edx, %xmm1, %xmm0 +; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpsubb %xmm0, %xmm1, %xmm1 +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpsubb {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm2 +; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm2 +; AVX-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpshufb %xmm0, %xmm3, %xmm0 +; AVX-NEXT: vpaddb %xmm2, %xmm0, %xmm0 ; AVX-NEXT: retq %out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %in, i1 0) ret <16 x i8> %out } -define <16 x i8> @testv16i8u(<16 x i8> %in) { +define <16 x i8> @testv16i8u(<16 x i8> %in) nounwind { ; SSE2-LABEL: testv16i8u: ; SSE2: # BB#0: -; SSE2: pushq %rbx -; SSE2: movaps %xmm0, -16(%rsp) -; SSE2-NEXT: movzbl -1(%rsp), %eax -; SSE2-NEXT: bsfl %eax, %eax -; SSE2-NEXT: movd %eax, %xmm0 -; SSE2-NEXT: movzbl -2(%rsp), %r11d -; SSE2-NEXT: movzbl -3(%rsp), %eax -; SSE2-NEXT: movzbl -4(%rsp), %r9d -; SSE2-NEXT: movzbl -5(%rsp), %edi -; SSE2-NEXT: movzbl -6(%rsp), %r10d -; SSE2-NEXT: movzbl -7(%rsp), %ecx -; SSE2-NEXT: movzbl -8(%rsp), %r8d -; SSE2-NEXT: movzbl -9(%rsp), %edx -; SSE2-NEXT: bsfl %edx, %edx -; SSE2-NEXT: movd %edx, %xmm1 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] -; SSE2-NEXT: bsfl %edi, %edx -; SSE2-NEXT: movd %edx, %xmm0 -; SSE2-NEXT: movzbl -10(%rsp), %edx -; SSE2-NEXT: movzbl -11(%rsp), %esi -; SSE2-NEXT: movzbl -12(%rsp), %edi -; SSE2-NEXT: movzbl -13(%rsp), %ebx -; SSE2-NEXT: bsfl %ebx, %ebx -; SSE2-NEXT: movd %ebx, %xmm2 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] -; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] -; SSE2-NEXT: bsfl %eax, %eax -; SSE2-NEXT: movd %eax, %xmm0 -; SSE2-NEXT: bsfl %esi, %eax -; SSE2-NEXT: movd %eax, %xmm3 -; SSE2-NEXT: punpcklbw %xmm0, %xmm3 # xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] -; SSE2-NEXT: bsfl %ecx, %eax -; SSE2-NEXT: movd %eax, %xmm0 -; SSE2-NEXT: movzbl -14(%rsp), %eax -; SSE2-NEXT: movzbl -15(%rsp), %ecx -; SSE2-NEXT: bsfl %ecx, %ecx -; SSE2-NEXT: movd %ecx, %xmm1 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] -; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7] -; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] -; SSE2-NEXT: bsfl %r11d, %ecx -; SSE2-NEXT: movd %ecx, %xmm0 -; SSE2-NEXT: bsfl %edx, %ecx -; SSE2-NEXT: movd %ecx, %xmm2 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] -; SSE2-NEXT: bsfl %r10d, %ecx -; SSE2-NEXT: movd %ecx, %xmm0 -; SSE2-NEXT: bsfl %eax, %eax -; SSE2-NEXT: movd %eax, %xmm3 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] -; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] -; SSE2-NEXT: bsfl %r9d, %eax -; SSE2-NEXT: movd %eax, %xmm0 -; SSE2-NEXT: bsfl %edi, %eax -; SSE2-NEXT: movd %eax, %xmm2 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] -; SSE2-NEXT: bsfl %r8d, %eax -; SSE2-NEXT: movd %eax, %xmm4 -; SSE2-NEXT: movzbl -16(%rsp), %eax -; SSE2-NEXT: bsfl %eax, %eax -; SSE2-NEXT: movd %eax, %xmm0 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7] -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] -; SSE2-NEXT: popq %rbx +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: psubb %xmm0, %xmm1 +; SSE2-NEXT: pand %xmm0, %xmm1 +; SSE2-NEXT: psubb {{.*}}(%rip), %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: psrlw $1, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: psubb %xmm0, %xmm1 +; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51] +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: pand %xmm0, %xmm2 +; SSE2-NEXT: psrlw $2, %xmm1 +; SSE2-NEXT: pand %xmm0, %xmm1 +; SSE2-NEXT: paddb %xmm2, %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: psrlw $4, %xmm0 +; SSE2-NEXT: paddb %xmm1, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 ; SSE2-NEXT: retq ; ; SSE3-LABEL: testv16i8u: ; SSE3: # BB#0: -; SSE3: pushq %rbx -; SSE3: movaps %xmm0, -16(%rsp) -; SSE3-NEXT: movzbl -1(%rsp), %eax -; SSE3-NEXT: bsfl %eax, %eax -; SSE3-NEXT: movd %eax, %xmm0 -; SSE3-NEXT: movzbl -2(%rsp), %r11d -; SSE3-NEXT: movzbl -3(%rsp), %eax -; SSE3-NEXT: movzbl -4(%rsp), %r9d -; SSE3-NEXT: movzbl -5(%rsp), %edi -; SSE3-NEXT: movzbl -6(%rsp), %r10d -; SSE3-NEXT: movzbl -7(%rsp), %ecx -; SSE3-NEXT: movzbl -8(%rsp), %r8d -; SSE3-NEXT: movzbl -9(%rsp), %edx -; SSE3-NEXT: bsfl %edx, %edx -; SSE3-NEXT: movd %edx, %xmm1 -; SSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] -; SSE3-NEXT: bsfl %edi, %edx -; SSE3-NEXT: movd %edx, %xmm0 -; SSE3-NEXT: movzbl -10(%rsp), %edx -; SSE3-NEXT: movzbl -11(%rsp), %esi -; SSE3-NEXT: movzbl -12(%rsp), %edi -; SSE3-NEXT: movzbl -13(%rsp), %ebx -; SSE3-NEXT: bsfl %ebx, %ebx -; SSE3-NEXT: movd %ebx, %xmm2 -; SSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] -; SSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] -; SSE3-NEXT: bsfl %eax, %eax -; SSE3-NEXT: movd %eax, %xmm0 -; SSE3-NEXT: bsfl %esi, %eax -; SSE3-NEXT: movd %eax, %xmm3 -; SSE3-NEXT: punpcklbw %xmm0, %xmm3 # xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] -; SSE3-NEXT: bsfl %ecx, %eax -; SSE3-NEXT: movd %eax, %xmm0 -; SSE3-NEXT: movzbl -14(%rsp), %eax -; SSE3-NEXT: movzbl -15(%rsp), %ecx -; SSE3-NEXT: bsfl %ecx, %ecx -; SSE3-NEXT: movd %ecx, %xmm1 -; SSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] -; SSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7] -; SSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] -; SSE3-NEXT: bsfl %r11d, %ecx -; SSE3-NEXT: movd %ecx, %xmm0 -; SSE3-NEXT: bsfl %edx, %ecx -; SSE3-NEXT: movd %ecx, %xmm2 -; SSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] -; SSE3-NEXT: bsfl %r10d, %ecx -; SSE3-NEXT: movd %ecx, %xmm0 -; SSE3-NEXT: bsfl %eax, %eax -; SSE3-NEXT: movd %eax, %xmm3 -; SSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] -; SSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] -; SSE3-NEXT: bsfl %r9d, %eax -; SSE3-NEXT: movd %eax, %xmm0 -; SSE3-NEXT: bsfl %edi, %eax -; SSE3-NEXT: movd %eax, %xmm2 -; SSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] -; SSE3-NEXT: bsfl %r8d, %eax -; SSE3-NEXT: movd %eax, %xmm4 -; SSE3-NEXT: movzbl -16(%rsp), %eax -; SSE3-NEXT: bsfl %eax, %eax -; SSE3-NEXT: movd %eax, %xmm0 -; SSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7] -; SSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; SSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] -; SSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] -; SSE3-NEXT: popq %rbx +; SSE3-NEXT: pxor %xmm1, %xmm1 +; SSE3-NEXT: psubb %xmm0, %xmm1 +; SSE3-NEXT: pand %xmm0, %xmm1 +; SSE3-NEXT: psubb {{.*}}(%rip), %xmm1 +; SSE3-NEXT: movdqa %xmm1, %xmm0 +; SSE3-NEXT: psrlw $1, %xmm0 +; SSE3-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE3-NEXT: psubb %xmm0, %xmm1 +; SSE3-NEXT: movdqa {{.*#+}} xmm0 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51] +; SSE3-NEXT: movdqa %xmm1, %xmm2 +; SSE3-NEXT: pand %xmm0, %xmm2 +; SSE3-NEXT: psrlw $2, %xmm1 +; SSE3-NEXT: pand %xmm0, %xmm1 +; SSE3-NEXT: paddb %xmm2, %xmm1 +; SSE3-NEXT: movdqa %xmm1, %xmm0 +; SSE3-NEXT: psrlw $4, %xmm0 +; SSE3-NEXT: paddb %xmm1, %xmm0 +; SSE3-NEXT: pand {{.*}}(%rip), %xmm0 ; SSE3-NEXT: retq ; ; SSSE3-LABEL: testv16i8u: ; SSSE3: # BB#0: -; SSSE3: pushq %rbx -; SSSE3: movaps %xmm0, -16(%rsp) -; SSSE3-NEXT: movzbl -1(%rsp), %eax -; SSSE3-NEXT: bsfl %eax, %eax -; SSSE3-NEXT: movd %eax, %xmm0 -; SSSE3-NEXT: movzbl -2(%rsp), %r11d -; SSSE3-NEXT: movzbl -3(%rsp), %eax -; SSSE3-NEXT: movzbl -4(%rsp), %r9d -; SSSE3-NEXT: movzbl -5(%rsp), %edi -; SSSE3-NEXT: movzbl -6(%rsp), %r10d -; SSSE3-NEXT: movzbl -7(%rsp), %ecx -; SSSE3-NEXT: movzbl -8(%rsp), %r8d -; SSSE3-NEXT: movzbl -9(%rsp), %edx -; SSSE3-NEXT: bsfl %edx, %edx -; SSSE3-NEXT: movd %edx, %xmm1 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] -; SSSE3-NEXT: bsfl %edi, %edx -; SSSE3-NEXT: movd %edx, %xmm0 -; SSSE3-NEXT: movzbl -10(%rsp), %edx -; SSSE3-NEXT: movzbl -11(%rsp), %esi -; SSSE3-NEXT: movzbl -12(%rsp), %edi -; SSSE3-NEXT: movzbl -13(%rsp), %ebx -; SSSE3-NEXT: bsfl %ebx, %ebx -; SSSE3-NEXT: movd %ebx, %xmm2 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] -; SSSE3-NEXT: bsfl %eax, %eax -; SSSE3-NEXT: movd %eax, %xmm0 -; SSSE3-NEXT: bsfl %esi, %eax -; SSSE3-NEXT: movd %eax, %xmm3 -; SSSE3-NEXT: punpcklbw %xmm0, %xmm3 # xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] -; SSSE3-NEXT: bsfl %ecx, %eax -; SSSE3-NEXT: movd %eax, %xmm0 -; SSSE3-NEXT: movzbl -14(%rsp), %eax -; SSSE3-NEXT: movzbl -15(%rsp), %ecx -; SSSE3-NEXT: bsfl %ecx, %ecx -; SSSE3-NEXT: movd %ecx, %xmm1 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7] -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] -; SSSE3-NEXT: bsfl %r11d, %ecx -; SSSE3-NEXT: movd %ecx, %xmm0 -; SSSE3-NEXT: bsfl %edx, %ecx -; SSSE3-NEXT: movd %ecx, %xmm2 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] -; SSSE3-NEXT: bsfl %r10d, %ecx -; SSSE3-NEXT: movd %ecx, %xmm0 -; SSSE3-NEXT: bsfl %eax, %eax -; SSSE3-NEXT: movd %eax, %xmm3 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] -; SSSE3-NEXT: bsfl %r9d, %eax -; SSSE3-NEXT: movd %eax, %xmm0 -; SSSE3-NEXT: bsfl %edi, %eax -; SSSE3-NEXT: movd %eax, %xmm2 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] -; SSSE3-NEXT: bsfl %r8d, %eax -; SSSE3-NEXT: movd %eax, %xmm4 -; SSSE3-NEXT: movzbl -16(%rsp), %eax -; SSSE3-NEXT: bsfl %eax, %eax -; SSSE3-NEXT: movd %eax, %xmm0 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7] -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] -; SSSE3-NEXT: popq %rbx +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: psubb %xmm0, %xmm1 +; SSSE3-NEXT: pand %xmm0, %xmm1 +; SSSE3-NEXT: psubb {{.*}}(%rip), %xmm1 +; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSSE3-NEXT: movdqa %xmm1, %xmm3 +; SSSE3-NEXT: pand %xmm2, %xmm3 +; SSSE3-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; SSSE3-NEXT: movdqa %xmm0, %xmm4 +; SSSE3-NEXT: pshufb %xmm3, %xmm4 +; SSSE3-NEXT: psrlw $4, %xmm1 +; SSSE3-NEXT: pand %xmm2, %xmm1 +; SSSE3-NEXT: pshufb %xmm1, %xmm0 +; SSSE3-NEXT: paddb %xmm4, %xmm0 ; SSSE3-NEXT: retq ; ; SSE41-LABEL: testv16i8u: ; SSE41: # BB#0: -; SSE41-NEXT: pextrb $1, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: pextrb $0, %xmm0, %ecx -; SSE41-NEXT: bsfl %ecx, %ecx -; SSE41-NEXT: movd %ecx, %xmm1 -; SSE41-NEXT: pinsrb $1, %eax, %xmm1 -; SSE41-NEXT: pextrb $2, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: pinsrb $2, %eax, %xmm1 -; SSE41-NEXT: pextrb $3, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: pinsrb $3, %eax, %xmm1 -; SSE41-NEXT: pextrb $4, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: pinsrb $4, %eax, %xmm1 -; SSE41-NEXT: pextrb $5, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: pinsrb $5, %eax, %xmm1 -; SSE41-NEXT: pextrb $6, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: pinsrb $6, %eax, %xmm1 -; SSE41-NEXT: pextrb $7, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: pinsrb $7, %eax, %xmm1 -; SSE41-NEXT: pextrb $8, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: pinsrb $8, %eax, %xmm1 -; SSE41-NEXT: pextrb $9, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: pinsrb $9, %eax, %xmm1 -; SSE41-NEXT: pextrb $10, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: pinsrb $10, %eax, %xmm1 -; SSE41-NEXT: pextrb $11, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: pinsrb $11, %eax, %xmm1 -; SSE41-NEXT: pextrb $12, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: pinsrb $12, %eax, %xmm1 -; SSE41-NEXT: pextrb $13, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: pinsrb $13, %eax, %xmm1 -; SSE41-NEXT: pextrb $14, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: pinsrb $14, %eax, %xmm1 -; SSE41-NEXT: pextrb $15, %xmm0, %eax -; SSE41-NEXT: bsfl %eax, %eax -; SSE41-NEXT: pinsrb $15, %eax, %xmm1 -; SSE41-NEXT: movdqa %xmm1, %xmm0 +; SSE41-NEXT: pxor %xmm1, %xmm1 +; SSE41-NEXT: psubb %xmm0, %xmm1 +; SSE41-NEXT: pand %xmm0, %xmm1 +; SSE41-NEXT: psubb {{.*}}(%rip), %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSE41-NEXT: movdqa %xmm1, %xmm3 +; SSE41-NEXT: pand %xmm2, %xmm3 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; SSE41-NEXT: movdqa %xmm0, %xmm4 +; SSE41-NEXT: pshufb %xmm3, %xmm4 +; SSE41-NEXT: psrlw $4, %xmm1 +; SSE41-NEXT: pand %xmm2, %xmm1 +; SSE41-NEXT: pshufb %xmm1, %xmm0 +; SSE41-NEXT: paddb %xmm4, %xmm0 ; SSE41-NEXT: retq ; ; AVX-LABEL: testv16i8u: ; AVX: # BB#0: -; AVX-NEXT: vpextrb $1, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vpextrb $0, %xmm0, %ecx -; AVX-NEXT: bsfl %ecx, %ecx -; AVX-NEXT: vmovd %ecx, %xmm1 -; AVX-NEXT: vpinsrb $1, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $2, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vpinsrb $2, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $3, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vpinsrb $3, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $4, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vpinsrb $4, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $5, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vpinsrb $5, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $6, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vpinsrb $6, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $7, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vpinsrb $7, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $8, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vpinsrb $8, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $9, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vpinsrb $9, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $10, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vpinsrb $10, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $11, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vpinsrb $11, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $12, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vpinsrb $12, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $13, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vpinsrb $13, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $14, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vpinsrb $14, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrb $15, %xmm0, %eax -; AVX-NEXT: bsfl %eax, %eax -; AVX-NEXT: vpinsrb $15, %eax, %xmm1, %xmm0 +; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpsubb %xmm0, %xmm1, %xmm1 +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpsubb {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm2 +; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm2 +; AVX-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpshufb %xmm0, %xmm3, %xmm0 +; AVX-NEXT: vpaddb %xmm2, %xmm0, %xmm0 ; AVX-NEXT: retq %out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %in, i1 -1) ret <16 x i8> %out } -define <2 x i64> @foldv2i64() { +define <2 x i64> @foldv2i64() nounwind { ; SSE-LABEL: foldv2i64: ; SSE: # BB#0: ; SSE-NEXT: movl $8, %eax @@ -1682,7 +915,7 @@ define <2 x i64> @foldv2i64() { ret <2 x i64> %out } -define <2 x i64> @foldv2i64u() { +define <2 x i64> @foldv2i64u() nounwind { ; SSE-LABEL: foldv2i64u: ; SSE: # BB#0: ; SSE-NEXT: movl $8, %eax @@ -1698,7 +931,7 @@ define <2 x i64> @foldv2i64u() { ret <2 x i64> %out } -define <4 x i32> @foldv4i32() { +define <4 x i32> @foldv4i32() nounwind { ; SSE-LABEL: foldv4i32: ; SSE: # BB#0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [8,0,32,0] @@ -1712,7 +945,7 @@ define <4 x i32> @foldv4i32() { ret <4 x i32> %out } -define <4 x i32> @foldv4i32u() { +define <4 x i32> @foldv4i32u() nounwind { ; SSE-LABEL: foldv4i32u: ; SSE: # BB#0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [8,0,32,0] @@ -1726,7 +959,7 @@ define <4 x i32> @foldv4i32u() { ret <4 x i32> %out } -define <8 x i16> @foldv8i16() { +define <8 x i16> @foldv8i16() nounwind { ; SSE-LABEL: foldv8i16: ; SSE: # BB#0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [8,0,16,0,16,0,3,3] @@ -1740,7 +973,7 @@ define <8 x i16> @foldv8i16() { ret <8 x i16> %out } -define <8 x i16> @foldv8i16u() { +define <8 x i16> @foldv8i16u() nounwind { ; SSE-LABEL: foldv8i16u: ; SSE: # BB#0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [8,0,16,0,16,0,3,3] @@ -1754,7 +987,7 @@ define <8 x i16> @foldv8i16u() { ret <8 x i16> %out } -define <16 x i8> @foldv16i8() { +define <16 x i8> @foldv16i8() nounwind { ; SSE-LABEL: foldv16i8: ; SSE: # BB#0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5] @@ -1768,7 +1001,7 @@ define <16 x i8> @foldv16i8() { ret <16 x i8> %out } -define <16 x i8> @foldv16i8u() { +define <16 x i8> @foldv16i8u() nounwind { ; SSE-LABEL: foldv16i8u: ; SSE: # BB#0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5] diff --git a/test/CodeGen/X86/vector-tzcnt-256.ll b/test/CodeGen/X86/vector-tzcnt-256.ll index 8f744f79f85f..a9ee27cc51f0 100644 --- a/test/CodeGen/X86/vector-tzcnt-256.ll +++ b/test/CodeGen/X86/vector-tzcnt-256.ll @@ -1,1190 +1,525 @@ -; RUN: llc < %s -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 -; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2 -target triple = "x86_64-unknown-unknown" - -define <4 x i64> @testv4i64(<4 x i64> %in) { +define <4 x i64> @testv4i64(<4 x i64> %in) nounwind { ; AVX1-LABEL: testv4i64: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX1-NEXT: vpextrq $1, %xmm1, %rax -; AVX1-NEXT: bsfq %rax, %rax -; AVX1-NEXT: movl $64, %ecx -; AVX1-NEXT: cmoveq %rcx, %rax -; AVX1-NEXT: vmovq %rax, %xmm2 -; AVX1-NEXT: vmovq %xmm1, %rax -; AVX1-NEXT: bsfq %rax, %rax -; AVX1-NEXT: cmoveq %rcx, %rax -; AVX1-NEXT: vmovq %rax, %xmm1 -; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] -; AVX1-NEXT: vpextrq $1, %xmm0, %rax -; AVX1-NEXT: bsfq %rax, %rax -; AVX1-NEXT: cmoveq %rcx, %rax -; AVX1-NEXT: vmovq %rax, %xmm2 -; AVX1-NEXT: vmovq %xmm0, %rax -; AVX1-NEXT: bsfq %rax, %rax -; AVX1-NEXT: cmoveq %rcx, %rax -; AVX1-NEXT: vmovq %rax, %xmm0 -; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpsubq %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vpsubq %xmm0, %xmm2, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1 +; AVX1-NEXT: vandps %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [1,1] +; AVX1-NEXT: vpsubq %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vpand %xmm4, %xmm1, %xmm5 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX1-NEXT: vpshufb %xmm5, %xmm6, %xmm5 +; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1 +; AVX1-NEXT: vpand %xmm4, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb %xmm1, %xmm6, %xmm1 +; AVX1-NEXT: vpaddb %xmm5, %xmm1, %xmm1 +; AVX1-NEXT: vpsadbw %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vpsubq %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm4, %xmm0, %xmm3 +; AVX1-NEXT: vpshufb %xmm3, %xmm6, %xmm3 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0 +; AVX1-NEXT: vpshufb %xmm0, %xmm6, %xmm0 +; AVX1-NEXT: vpaddb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpsadbw %xmm2, %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: testv4i64: ; AVX2: # BB#0: -; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 -; AVX2-NEXT: vpextrq $1, %xmm1, %rax -; AVX2-NEXT: bsfq %rax, %rax -; AVX2-NEXT: movl $64, %ecx -; AVX2-NEXT: cmoveq %rcx, %rax -; AVX2-NEXT: vmovq %rax, %xmm2 -; AVX2-NEXT: vmovq %xmm1, %rax -; AVX2-NEXT: bsfq %rax, %rax -; AVX2-NEXT: cmoveq %rcx, %rax -; AVX2-NEXT: vmovq %rax, %xmm1 -; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] -; AVX2-NEXT: vpextrq $1, %xmm0, %rax -; AVX2-NEXT: bsfq %rax, %rax -; AVX2-NEXT: cmoveq %rcx, %rax -; AVX2-NEXT: vmovq %rax, %xmm2 -; AVX2-NEXT: vmovq %xmm0, %rax -; AVX2-NEXT: bsfq %rax, %rax -; AVX2-NEXT: cmoveq %rcx, %rax -; AVX2-NEXT: vmovq %rax, %xmm0 -; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] -; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: vpxor %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vpsubq %ymm0, %ymm1, %ymm2 +; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpsubq %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm3 +; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0 +; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0 +; AVX2-NEXT: vpaddb %ymm3, %ymm0, %ymm0 +; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq %out = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %in, i1 0) ret <4 x i64> %out } -define <4 x i64> @testv4i64u(<4 x i64> %in) { +define <4 x i64> @testv4i64u(<4 x i64> %in) nounwind { ; AVX1-LABEL: testv4i64u: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX1-NEXT: vpextrq $1, %xmm1, %rax -; AVX1-NEXT: bsfq %rax, %rax -; AVX1-NEXT: vmovq %rax, %xmm2 -; AVX1-NEXT: vmovq %xmm1, %rax -; AVX1-NEXT: bsfq %rax, %rax -; AVX1-NEXT: vmovq %rax, %xmm1 -; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] -; AVX1-NEXT: vpextrq $1, %xmm0, %rax -; AVX1-NEXT: bsfq %rax, %rax -; AVX1-NEXT: vmovq %rax, %xmm2 -; AVX1-NEXT: vmovq %xmm0, %rax -; AVX1-NEXT: bsfq %rax, %rax -; AVX1-NEXT: vmovq %rax, %xmm0 -; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpsubq %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vpsubq %xmm0, %xmm2, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1 +; AVX1-NEXT: vandps %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [1,1] +; AVX1-NEXT: vpsubq %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vpand %xmm4, %xmm1, %xmm5 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX1-NEXT: vpshufb %xmm5, %xmm6, %xmm5 +; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1 +; AVX1-NEXT: vpand %xmm4, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb %xmm1, %xmm6, %xmm1 +; AVX1-NEXT: vpaddb %xmm5, %xmm1, %xmm1 +; AVX1-NEXT: vpsadbw %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vpsubq %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm4, %xmm0, %xmm3 +; AVX1-NEXT: vpshufb %xmm3, %xmm6, %xmm3 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0 +; AVX1-NEXT: vpshufb %xmm0, %xmm6, %xmm0 +; AVX1-NEXT: vpaddb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpsadbw %xmm2, %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: testv4i64u: ; AVX2: # BB#0: -; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 -; AVX2-NEXT: vpextrq $1, %xmm1, %rax -; AVX2-NEXT: bsfq %rax, %rax -; AVX2-NEXT: vmovq %rax, %xmm2 -; AVX2-NEXT: vmovq %xmm1, %rax -; AVX2-NEXT: bsfq %rax, %rax -; AVX2-NEXT: vmovq %rax, %xmm1 -; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] -; AVX2-NEXT: vpextrq $1, %xmm0, %rax -; AVX2-NEXT: bsfq %rax, %rax -; AVX2-NEXT: vmovq %rax, %xmm2 -; AVX2-NEXT: vmovq %xmm0, %rax -; AVX2-NEXT: bsfq %rax, %rax -; AVX2-NEXT: vmovq %rax, %xmm0 -; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] -; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: vpxor %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vpsubq %ymm0, %ymm1, %ymm2 +; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpsubq %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm3 +; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0 +; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0 +; AVX2-NEXT: vpaddb %ymm3, %ymm0, %ymm0 +; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq %out = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %in, i1 -1) ret <4 x i64> %out } -define <8 x i32> @testv8i32(<8 x i32> %in) { +define <8 x i32> @testv8i32(<8 x i32> %in) nounwind { ; AVX1-LABEL: testv8i32: ; AVX1: # BB#0: -; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX1-NEXT: vpextrd $1, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %ecx -; AVX1-NEXT: movl $32, %eax -; AVX1-NEXT: cmovel %eax, %ecx -; AVX1-NEXT: vmovd %xmm1, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: vmovd %edx, %xmm2 -; AVX1-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrd $2, %xmm1, %ecx -; AVX1-NEXT: bsfl %ecx, %ecx -; AVX1-NEXT: cmovel %eax, %ecx -; AVX1-NEXT: vpinsrd $2, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrd $3, %xmm1, %ecx -; AVX1-NEXT: bsfl %ecx, %ecx -; AVX1-NEXT: cmovel %eax, %ecx -; AVX1-NEXT: vpinsrd $3, %ecx, %xmm2, %xmm1 -; AVX1-NEXT: vpextrd $1, %xmm0, %ecx -; AVX1-NEXT: bsfl %ecx, %ecx -; AVX1-NEXT: cmovel %eax, %ecx -; AVX1-NEXT: vmovd %xmm0, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: vmovd %edx, %xmm2 -; AVX1-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrd $2, %xmm0, %ecx -; AVX1-NEXT: bsfl %ecx, %ecx -; AVX1-NEXT: cmovel %eax, %ecx -; AVX1-NEXT: vpinsrd $2, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrd $3, %xmm0, %ecx -; AVX1-NEXT: bsfl %ecx, %ecx -; AVX1-NEXT: cmovel %eax, %ecx -; AVX1-NEXT: vpinsrd $3, %ecx, %xmm2, %xmm0 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpsubd %xmm2, %xmm1, %xmm2 +; AVX1-NEXT: vpsubd %xmm0, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vandps %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [1,1,1,1] +; AVX1-NEXT: vpsubd %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vpand %xmm4, %xmm2, %xmm5 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX1-NEXT: vpshufb %xmm5, %xmm6, %xmm5 +; AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2 +; AVX1-NEXT: vpand %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vpshufb %xmm2, %xmm6, %xmm2 +; AVX1-NEXT: vpaddb %xmm5, %xmm2, %xmm2 +; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; AVX1-NEXT: vpsadbw %xmm1, %xmm5, %xmm5 +; AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; AVX1-NEXT: vpsadbw %xmm1, %xmm2, %xmm2 +; AVX1-NEXT: vpackuswb %xmm5, %xmm2, %xmm2 +; AVX1-NEXT: vpsubd %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm4, %xmm0, %xmm3 +; AVX1-NEXT: vpshufb %xmm3, %xmm6, %xmm3 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0 +; AVX1-NEXT: vpshufb %xmm0, %xmm6, %xmm0 +; AVX1-NEXT: vpaddb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX1-NEXT: vpsadbw %xmm1, %xmm3, %xmm3 +; AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpackuswb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: testv8i32: ; AVX2: # BB#0: -; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 -; AVX2-NEXT: vpextrd $1, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %ecx -; AVX2-NEXT: movl $32, %eax -; AVX2-NEXT: cmovel %eax, %ecx -; AVX2-NEXT: vmovd %xmm1, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: vmovd %edx, %xmm2 -; AVX2-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrd $2, %xmm1, %ecx -; AVX2-NEXT: bsfl %ecx, %ecx -; AVX2-NEXT: cmovel %eax, %ecx -; AVX2-NEXT: vpinsrd $2, %ecx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrd $3, %xmm1, %ecx -; AVX2-NEXT: bsfl %ecx, %ecx -; AVX2-NEXT: cmovel %eax, %ecx -; AVX2-NEXT: vpinsrd $3, %ecx, %xmm2, %xmm1 -; AVX2-NEXT: vpextrd $1, %xmm0, %ecx -; AVX2-NEXT: bsfl %ecx, %ecx -; AVX2-NEXT: cmovel %eax, %ecx -; AVX2-NEXT: vmovd %xmm0, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: vmovd %edx, %xmm2 -; AVX2-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrd $2, %xmm0, %ecx -; AVX2-NEXT: bsfl %ecx, %ecx -; AVX2-NEXT: cmovel %eax, %ecx -; AVX2-NEXT: vpinsrd $2, %ecx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrd $3, %xmm0, %ecx -; AVX2-NEXT: bsfl %ecx, %ecx -; AVX2-NEXT: cmovel %eax, %ecx -; AVX2-NEXT: vpinsrd $3, %ecx, %xmm2, %xmm0 -; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: vpxor %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vpsubd %ymm0, %ymm1, %ymm2 +; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpsubd %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm3 +; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0 +; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0 +; AVX2-NEXT: vpaddb %ymm3, %ymm0, %ymm0 +; AVX2-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] +; AVX2-NEXT: vpsadbw %ymm1, %ymm2, %ymm2 +; AVX2-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] +; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: retq %out = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %in, i1 0) ret <8 x i32> %out } -define <8 x i32> @testv8i32u(<8 x i32> %in) { +define <8 x i32> @testv8i32u(<8 x i32> %in) nounwind { ; AVX1-LABEL: testv8i32u: ; AVX1: # BB#0: -; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX1-NEXT: vpextrd $1, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vmovd %xmm1, %ecx -; AVX1-NEXT: bsfl %ecx, %ecx -; AVX1-NEXT: vmovd %ecx, %xmm2 -; AVX1-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrd $2, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrd $3, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrd $3, %eax, %xmm2, %xmm1 -; AVX1-NEXT: vpextrd $1, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vmovd %xmm0, %ecx -; AVX1-NEXT: bsfl %ecx, %ecx -; AVX1-NEXT: vmovd %ecx, %xmm2 -; AVX1-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrd $2, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrd $3, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpsubd %xmm2, %xmm1, %xmm2 +; AVX1-NEXT: vpsubd %xmm0, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vandps %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [1,1,1,1] +; AVX1-NEXT: vpsubd %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vpand %xmm4, %xmm2, %xmm5 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX1-NEXT: vpshufb %xmm5, %xmm6, %xmm5 +; AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2 +; AVX1-NEXT: vpand %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vpshufb %xmm2, %xmm6, %xmm2 +; AVX1-NEXT: vpaddb %xmm5, %xmm2, %xmm2 +; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; AVX1-NEXT: vpsadbw %xmm1, %xmm5, %xmm5 +; AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; AVX1-NEXT: vpsadbw %xmm1, %xmm2, %xmm2 +; AVX1-NEXT: vpackuswb %xmm5, %xmm2, %xmm2 +; AVX1-NEXT: vpsubd %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm4, %xmm0, %xmm3 +; AVX1-NEXT: vpshufb %xmm3, %xmm6, %xmm3 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0 +; AVX1-NEXT: vpshufb %xmm0, %xmm6, %xmm0 +; AVX1-NEXT: vpaddb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX1-NEXT: vpsadbw %xmm1, %xmm3, %xmm3 +; AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpackuswb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: testv8i32u: ; AVX2: # BB#0: -; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 -; AVX2-NEXT: vpextrd $1, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vmovd %xmm1, %ecx -; AVX2-NEXT: bsfl %ecx, %ecx -; AVX2-NEXT: vmovd %ecx, %xmm2 -; AVX2-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrd $2, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrd $3, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrd $3, %eax, %xmm2, %xmm1 -; AVX2-NEXT: vpextrd $1, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vmovd %xmm0, %ecx -; AVX2-NEXT: bsfl %ecx, %ecx -; AVX2-NEXT: vmovd %ecx, %xmm2 -; AVX2-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrd $2, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrd $3, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0 -; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: vpxor %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vpsubd %ymm0, %ymm1, %ymm2 +; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpsubd %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm3 +; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0 +; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0 +; AVX2-NEXT: vpaddb %ymm3, %ymm0, %ymm0 +; AVX2-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] +; AVX2-NEXT: vpsadbw %ymm1, %ymm2, %ymm2 +; AVX2-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] +; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: retq %out = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %in, i1 -1) ret <8 x i32> %out } -define <16 x i16> @testv16i16(<16 x i16> %in) { +define <16 x i16> @testv16i16(<16 x i16> %in) nounwind { ; AVX1-LABEL: testv16i16: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX1-NEXT: vpextrw $1, %xmm1, %eax -; AVX1-NEXT: bsfw %ax, %cx -; AVX1-NEXT: movw $16, %ax -; AVX1-NEXT: cmovew %ax, %cx -; AVX1-NEXT: vmovd %xmm1, %edx -; AVX1-NEXT: bsfw %dx, %dx -; AVX1-NEXT: cmovew %ax, %dx -; AVX1-NEXT: vmovd %edx, %xmm2 -; AVX1-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $2, %xmm1, %ecx -; AVX1-NEXT: bsfw %cx, %cx -; AVX1-NEXT: cmovew %ax, %cx -; AVX1-NEXT: vpinsrw $2, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $3, %xmm1, %ecx -; AVX1-NEXT: bsfw %cx, %cx -; AVX1-NEXT: cmovew %ax, %cx -; AVX1-NEXT: vpinsrw $3, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $4, %xmm1, %ecx -; AVX1-NEXT: bsfw %cx, %cx -; AVX1-NEXT: cmovew %ax, %cx -; AVX1-NEXT: vpinsrw $4, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $5, %xmm1, %ecx -; AVX1-NEXT: bsfw %cx, %cx -; AVX1-NEXT: cmovew %ax, %cx -; AVX1-NEXT: vpinsrw $5, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $6, %xmm1, %ecx -; AVX1-NEXT: bsfw %cx, %cx -; AVX1-NEXT: cmovew %ax, %cx -; AVX1-NEXT: vpinsrw $6, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $7, %xmm1, %ecx -; AVX1-NEXT: bsfw %cx, %cx -; AVX1-NEXT: cmovew %ax, %cx -; AVX1-NEXT: vpinsrw $7, %ecx, %xmm2, %xmm1 -; AVX1-NEXT: vpextrw $1, %xmm0, %ecx -; AVX1-NEXT: bsfw %cx, %cx -; AVX1-NEXT: cmovew %ax, %cx -; AVX1-NEXT: vmovd %xmm0, %edx -; AVX1-NEXT: bsfw %dx, %dx -; AVX1-NEXT: cmovew %ax, %dx -; AVX1-NEXT: vmovd %edx, %xmm2 -; AVX1-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $2, %xmm0, %ecx -; AVX1-NEXT: bsfw %cx, %cx -; AVX1-NEXT: cmovew %ax, %cx -; AVX1-NEXT: vpinsrw $2, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $3, %xmm0, %ecx -; AVX1-NEXT: bsfw %cx, %cx -; AVX1-NEXT: cmovew %ax, %cx -; AVX1-NEXT: vpinsrw $3, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $4, %xmm0, %ecx -; AVX1-NEXT: bsfw %cx, %cx -; AVX1-NEXT: cmovew %ax, %cx -; AVX1-NEXT: vpinsrw $4, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $5, %xmm0, %ecx -; AVX1-NEXT: bsfw %cx, %cx -; AVX1-NEXT: cmovew %ax, %cx -; AVX1-NEXT: vpinsrw $5, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $6, %xmm0, %ecx -; AVX1-NEXT: bsfw %cx, %cx -; AVX1-NEXT: cmovew %ax, %cx -; AVX1-NEXT: vpinsrw $6, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $7, %xmm0, %ecx -; AVX1-NEXT: bsfw %cx, %cx -; AVX1-NEXT: cmovew %ax, %cx -; AVX1-NEXT: vpinsrw $7, %ecx, %xmm2, %xmm0 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpsubw %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vpsubw %xmm0, %xmm2, %xmm2 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 +; AVX1-NEXT: vandps %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm1 = [1,1,1,1,1,1,1,1] +; AVX1-NEXT: vpsubw %xmm1, %xmm0, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX1-NEXT: vpshufb %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2 +; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2 +; AVX1-NEXT: vpaddb %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vpsllw $8, %xmm2, %xmm4 +; AVX1-NEXT: vpaddb %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vpsubw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm1 +; AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0 +; AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpsllw $8, %xmm0, %xmm1 +; AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: testv16i16: ; AVX2: # BB#0: -; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 -; AVX2-NEXT: vpextrw $1, %xmm1, %eax -; AVX2-NEXT: bsfw %ax, %cx -; AVX2-NEXT: movw $16, %ax -; AVX2-NEXT: cmovew %ax, %cx -; AVX2-NEXT: vmovd %xmm1, %edx -; AVX2-NEXT: bsfw %dx, %dx -; AVX2-NEXT: cmovew %ax, %dx -; AVX2-NEXT: vmovd %edx, %xmm2 -; AVX2-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $2, %xmm1, %ecx -; AVX2-NEXT: bsfw %cx, %cx -; AVX2-NEXT: cmovew %ax, %cx -; AVX2-NEXT: vpinsrw $2, %ecx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $3, %xmm1, %ecx -; AVX2-NEXT: bsfw %cx, %cx -; AVX2-NEXT: cmovew %ax, %cx -; AVX2-NEXT: vpinsrw $3, %ecx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $4, %xmm1, %ecx -; AVX2-NEXT: bsfw %cx, %cx -; AVX2-NEXT: cmovew %ax, %cx -; AVX2-NEXT: vpinsrw $4, %ecx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $5, %xmm1, %ecx -; AVX2-NEXT: bsfw %cx, %cx -; AVX2-NEXT: cmovew %ax, %cx -; AVX2-NEXT: vpinsrw $5, %ecx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $6, %xmm1, %ecx -; AVX2-NEXT: bsfw %cx, %cx -; AVX2-NEXT: cmovew %ax, %cx -; AVX2-NEXT: vpinsrw $6, %ecx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $7, %xmm1, %ecx -; AVX2-NEXT: bsfw %cx, %cx -; AVX2-NEXT: cmovew %ax, %cx -; AVX2-NEXT: vpinsrw $7, %ecx, %xmm2, %xmm1 -; AVX2-NEXT: vpextrw $1, %xmm0, %ecx -; AVX2-NEXT: bsfw %cx, %cx -; AVX2-NEXT: cmovew %ax, %cx -; AVX2-NEXT: vmovd %xmm0, %edx -; AVX2-NEXT: bsfw %dx, %dx -; AVX2-NEXT: cmovew %ax, %dx -; AVX2-NEXT: vmovd %edx, %xmm2 -; AVX2-NEXT: vpinsrw $1, %ecx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $2, %xmm0, %ecx -; AVX2-NEXT: bsfw %cx, %cx -; AVX2-NEXT: cmovew %ax, %cx -; AVX2-NEXT: vpinsrw $2, %ecx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $3, %xmm0, %ecx -; AVX2-NEXT: bsfw %cx, %cx -; AVX2-NEXT: cmovew %ax, %cx -; AVX2-NEXT: vpinsrw $3, %ecx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $4, %xmm0, %ecx -; AVX2-NEXT: bsfw %cx, %cx -; AVX2-NEXT: cmovew %ax, %cx -; AVX2-NEXT: vpinsrw $4, %ecx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $5, %xmm0, %ecx -; AVX2-NEXT: bsfw %cx, %cx -; AVX2-NEXT: cmovew %ax, %cx -; AVX2-NEXT: vpinsrw $5, %ecx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $6, %xmm0, %ecx -; AVX2-NEXT: bsfw %cx, %cx -; AVX2-NEXT: cmovew %ax, %cx -; AVX2-NEXT: vpinsrw $6, %ecx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $7, %xmm0, %ecx -; AVX2-NEXT: bsfw %cx, %cx -; AVX2-NEXT: cmovew %ax, %cx -; AVX2-NEXT: vpinsrw $7, %ecx, %xmm2, %xmm0 -; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: vpxor %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vpsubw %ymm0, %ymm1, %ymm1 +; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpsubw {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm2 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm2 +; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0 +; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpshufb %ymm0, %ymm3, %ymm0 +; AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpsllw $8, %ymm0, %ymm1 +; AVX2-NEXT: vpaddb %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0 ; AVX2-NEXT: retq %out = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %in, i1 0) ret <16 x i16> %out } -define <16 x i16> @testv16i16u(<16 x i16> %in) { +define <16 x i16> @testv16i16u(<16 x i16> %in) nounwind { ; AVX1-LABEL: testv16i16u: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX1-NEXT: vpextrw $1, %xmm1, %eax -; AVX1-NEXT: bsfw %ax, %ax -; AVX1-NEXT: vmovd %xmm1, %ecx -; AVX1-NEXT: bsfw %cx, %cx -; AVX1-NEXT: vmovd %ecx, %xmm2 -; AVX1-NEXT: vpinsrw $1, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $2, %xmm1, %eax -; AVX1-NEXT: bsfw %ax, %ax -; AVX1-NEXT: vpinsrw $2, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $3, %xmm1, %eax -; AVX1-NEXT: bsfw %ax, %ax -; AVX1-NEXT: vpinsrw $3, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $4, %xmm1, %eax -; AVX1-NEXT: bsfw %ax, %ax -; AVX1-NEXT: vpinsrw $4, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $5, %xmm1, %eax -; AVX1-NEXT: bsfw %ax, %ax -; AVX1-NEXT: vpinsrw $5, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $6, %xmm1, %eax -; AVX1-NEXT: bsfw %ax, %ax -; AVX1-NEXT: vpinsrw $6, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $7, %xmm1, %eax -; AVX1-NEXT: bsfw %ax, %ax -; AVX1-NEXT: vpinsrw $7, %eax, %xmm2, %xmm1 -; AVX1-NEXT: vpextrw $1, %xmm0, %eax -; AVX1-NEXT: bsfw %ax, %ax -; AVX1-NEXT: vmovd %xmm0, %ecx -; AVX1-NEXT: bsfw %cx, %cx -; AVX1-NEXT: vmovd %ecx, %xmm2 -; AVX1-NEXT: vpinsrw $1, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $2, %xmm0, %eax -; AVX1-NEXT: bsfw %ax, %ax -; AVX1-NEXT: vpinsrw $2, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $3, %xmm0, %eax -; AVX1-NEXT: bsfw %ax, %ax -; AVX1-NEXT: vpinsrw $3, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $4, %xmm0, %eax -; AVX1-NEXT: bsfw %ax, %ax -; AVX1-NEXT: vpinsrw $4, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $5, %xmm0, %eax -; AVX1-NEXT: bsfw %ax, %ax -; AVX1-NEXT: vpinsrw $5, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $6, %xmm0, %eax -; AVX1-NEXT: bsfw %ax, %ax -; AVX1-NEXT: vpinsrw $6, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrw $7, %xmm0, %eax -; AVX1-NEXT: bsfw %ax, %ax -; AVX1-NEXT: vpinsrw $7, %eax, %xmm2, %xmm0 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpsubw %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vpsubw %xmm0, %xmm2, %xmm2 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 +; AVX1-NEXT: vandps %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm1 = [1,1,1,1,1,1,1,1] +; AVX1-NEXT: vpsubw %xmm1, %xmm0, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm4 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX1-NEXT: vpshufb %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2 +; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2 +; AVX1-NEXT: vpaddb %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vpsllw $8, %xmm2, %xmm4 +; AVX1-NEXT: vpaddb %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vpsubw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm1 +; AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0 +; AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpsllw $8, %xmm0, %xmm1 +; AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: testv16i16u: ; AVX2: # BB#0: -; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 -; AVX2-NEXT: vpextrw $1, %xmm1, %eax -; AVX2-NEXT: bsfw %ax, %ax -; AVX2-NEXT: vmovd %xmm1, %ecx -; AVX2-NEXT: bsfw %cx, %cx -; AVX2-NEXT: vmovd %ecx, %xmm2 -; AVX2-NEXT: vpinsrw $1, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $2, %xmm1, %eax -; AVX2-NEXT: bsfw %ax, %ax -; AVX2-NEXT: vpinsrw $2, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $3, %xmm1, %eax -; AVX2-NEXT: bsfw %ax, %ax -; AVX2-NEXT: vpinsrw $3, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $4, %xmm1, %eax -; AVX2-NEXT: bsfw %ax, %ax -; AVX2-NEXT: vpinsrw $4, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $5, %xmm1, %eax -; AVX2-NEXT: bsfw %ax, %ax -; AVX2-NEXT: vpinsrw $5, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $6, %xmm1, %eax -; AVX2-NEXT: bsfw %ax, %ax -; AVX2-NEXT: vpinsrw $6, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $7, %xmm1, %eax -; AVX2-NEXT: bsfw %ax, %ax -; AVX2-NEXT: vpinsrw $7, %eax, %xmm2, %xmm1 -; AVX2-NEXT: vpextrw $1, %xmm0, %eax -; AVX2-NEXT: bsfw %ax, %ax -; AVX2-NEXT: vmovd %xmm0, %ecx -; AVX2-NEXT: bsfw %cx, %cx -; AVX2-NEXT: vmovd %ecx, %xmm2 -; AVX2-NEXT: vpinsrw $1, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $2, %xmm0, %eax -; AVX2-NEXT: bsfw %ax, %ax -; AVX2-NEXT: vpinsrw $2, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $3, %xmm0, %eax -; AVX2-NEXT: bsfw %ax, %ax -; AVX2-NEXT: vpinsrw $3, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $4, %xmm0, %eax -; AVX2-NEXT: bsfw %ax, %ax -; AVX2-NEXT: vpinsrw $4, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $5, %xmm0, %eax -; AVX2-NEXT: bsfw %ax, %ax -; AVX2-NEXT: vpinsrw $5, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $6, %xmm0, %eax -; AVX2-NEXT: bsfw %ax, %ax -; AVX2-NEXT: vpinsrw $6, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrw $7, %xmm0, %eax -; AVX2-NEXT: bsfw %ax, %ax -; AVX2-NEXT: vpinsrw $7, %eax, %xmm2, %xmm0 -; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: vpxor %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vpsubw %ymm0, %ymm1, %ymm1 +; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpsubw {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm2 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm2 +; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0 +; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpshufb %ymm0, %ymm3, %ymm0 +; AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpsllw $8, %ymm0, %ymm1 +; AVX2-NEXT: vpaddb %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0 ; AVX2-NEXT: retq %out = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %in, i1 -1) ret <16 x i16> %out } -define <32 x i8> @testv32i8(<32 x i8> %in) { +define <32 x i8> @testv32i8(<32 x i8> %in) nounwind { ; AVX1-LABEL: testv32i8: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX1-NEXT: vpextrb $1, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %edx -; AVX1-NEXT: movl $32, %eax -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: movl $8, %ecx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpextrb $0, %xmm1, %esi -; AVX1-NEXT: bsfl %esi, %esi -; AVX1-NEXT: cmovel %eax, %esi -; AVX1-NEXT: cmpl $32, %esi -; AVX1-NEXT: cmovel %ecx, %esi -; AVX1-NEXT: vmovd %esi, %xmm2 -; AVX1-NEXT: vpinsrb $1, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $2, %xmm1, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $2, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $3, %xmm1, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $3, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $4, %xmm1, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $4, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $5, %xmm1, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $5, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $6, %xmm1, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $6, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $7, %xmm1, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $7, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $8, %xmm1, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $8, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $9, %xmm1, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $9, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $10, %xmm1, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $10, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $11, %xmm1, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $11, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $12, %xmm1, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $12, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $13, %xmm1, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $13, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $14, %xmm1, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $14, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $15, %xmm1, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $15, %edx, %xmm2, %xmm1 -; AVX1-NEXT: vpextrb $1, %xmm0, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpextrb $0, %xmm0, %esi -; AVX1-NEXT: bsfl %esi, %esi -; AVX1-NEXT: cmovel %eax, %esi -; AVX1-NEXT: cmpl $32, %esi -; AVX1-NEXT: cmovel %ecx, %esi -; AVX1-NEXT: vmovd %esi, %xmm2 -; AVX1-NEXT: vpinsrb $1, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $2, %xmm0, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $2, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $3, %xmm0, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $3, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $4, %xmm0, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $4, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $5, %xmm0, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $5, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $6, %xmm0, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $6, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $7, %xmm0, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $7, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $8, %xmm0, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $8, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $9, %xmm0, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $9, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $10, %xmm0, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $10, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $11, %xmm0, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $11, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $12, %xmm0, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $12, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $13, %xmm0, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $13, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $14, %xmm0, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $14, %edx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $15, %xmm0, %edx -; AVX1-NEXT: bsfl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: cmpl $32, %edx -; AVX1-NEXT: cmovel %ecx, %edx -; AVX1-NEXT: vpinsrb $15, %edx, %xmm2, %xmm0 +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpsubb %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vpsubb %xmm0, %xmm2, %xmm2 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 +; AVX1-NEXT: vandps %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; AVX1-NEXT: vpsubb %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vpand %xmm3, %xmm1, %xmm4 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX1-NEXT: vpshufb %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1 +; AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1 +; AVX1-NEXT: vpaddb %xmm4, %xmm1, %xmm1 +; AVX1-NEXT: vpsubb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm2 +; AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0 +; AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: testv32i8: ; AVX2: # BB#0: -; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 -; AVX2-NEXT: vpextrb $1, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %edx -; AVX2-NEXT: movl $32, %eax -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: movl $8, %ecx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpextrb $0, %xmm1, %esi -; AVX2-NEXT: bsfl %esi, %esi -; AVX2-NEXT: cmovel %eax, %esi -; AVX2-NEXT: cmpl $32, %esi -; AVX2-NEXT: cmovel %ecx, %esi -; AVX2-NEXT: vmovd %esi, %xmm2 -; AVX2-NEXT: vpinsrb $1, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $2, %xmm1, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $2, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $3, %xmm1, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $3, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $4, %xmm1, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $4, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $5, %xmm1, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $5, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $6, %xmm1, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $6, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $7, %xmm1, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $7, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $8, %xmm1, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $8, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $9, %xmm1, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $9, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $10, %xmm1, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $10, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $11, %xmm1, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $11, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $12, %xmm1, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $12, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $13, %xmm1, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $13, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $14, %xmm1, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $14, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $15, %xmm1, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $15, %edx, %xmm2, %xmm1 -; AVX2-NEXT: vpextrb $1, %xmm0, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpextrb $0, %xmm0, %esi -; AVX2-NEXT: bsfl %esi, %esi -; AVX2-NEXT: cmovel %eax, %esi -; AVX2-NEXT: cmpl $32, %esi -; AVX2-NEXT: cmovel %ecx, %esi -; AVX2-NEXT: vmovd %esi, %xmm2 -; AVX2-NEXT: vpinsrb $1, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $2, %xmm0, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $2, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $3, %xmm0, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $3, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $4, %xmm0, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $4, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $5, %xmm0, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $5, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $6, %xmm0, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $6, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $7, %xmm0, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $7, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $8, %xmm0, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $8, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $9, %xmm0, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $9, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $10, %xmm0, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $10, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $11, %xmm0, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $11, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $12, %xmm0, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $12, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $13, %xmm0, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $13, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $14, %xmm0, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $14, %edx, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $15, %xmm0, %edx -; AVX2-NEXT: bsfl %edx, %edx -; AVX2-NEXT: cmovel %eax, %edx -; AVX2-NEXT: cmpl $32, %edx -; AVX2-NEXT: cmovel %ecx, %edx -; AVX2-NEXT: vpinsrb $15, %edx, %xmm2, %xmm0 -; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: vpxor %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vpsubb %ymm0, %ymm1, %ymm1 +; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpsubb {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm2 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm2 +; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0 +; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpshufb %ymm0, %ymm3, %ymm0 +; AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: retq %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %in, i1 0) ret <32 x i8> %out } -define <32 x i8> @testv32i8u(<32 x i8> %in) { +define <32 x i8> @testv32i8u(<32 x i8> %in) nounwind { ; AVX1-LABEL: testv32i8u: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX1-NEXT: vpextrb $1, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpextrb $0, %xmm1, %ecx -; AVX1-NEXT: bsfl %ecx, %ecx -; AVX1-NEXT: vmovd %ecx, %xmm2 -; AVX1-NEXT: vpinsrb $1, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $2, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $2, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $3, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $3, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $4, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $4, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $5, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $5, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $6, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $6, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $7, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $7, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $8, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $8, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $9, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $9, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $10, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $10, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $11, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $11, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $12, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $12, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $13, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $13, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $14, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $14, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $15, %xmm1, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $15, %eax, %xmm2, %xmm1 -; AVX1-NEXT: vpextrb $1, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpextrb $0, %xmm0, %ecx -; AVX1-NEXT: bsfl %ecx, %ecx -; AVX1-NEXT: vmovd %ecx, %xmm2 -; AVX1-NEXT: vpinsrb $1, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $2, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $2, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $3, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $3, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $4, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $4, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $5, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $5, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $6, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $6, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $7, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $7, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $8, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $8, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $9, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $9, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $10, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $10, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $11, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $11, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $12, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $12, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $13, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $13, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $14, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $14, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrb $15, %xmm0, %eax -; AVX1-NEXT: bsfl %eax, %eax -; AVX1-NEXT: vpinsrb $15, %eax, %xmm2, %xmm0 +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpsubb %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vpsubb %xmm0, %xmm2, %xmm2 +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 +; AVX1-NEXT: vandps %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; AVX1-NEXT: vpsubb %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vpand %xmm3, %xmm1, %xmm4 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX1-NEXT: vpshufb %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1 +; AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1 +; AVX1-NEXT: vpaddb %xmm4, %xmm1, %xmm1 +; AVX1-NEXT: vpsubb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm2 +; AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0 +; AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: testv32i8u: ; AVX2: # BB#0: -; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 -; AVX2-NEXT: vpextrb $1, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpextrb $0, %xmm1, %ecx -; AVX2-NEXT: bsfl %ecx, %ecx -; AVX2-NEXT: vmovd %ecx, %xmm2 -; AVX2-NEXT: vpinsrb $1, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $2, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $2, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $3, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $3, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $4, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $4, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $5, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $5, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $6, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $6, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $7, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $7, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $8, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $8, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $9, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $9, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $10, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $10, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $11, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $11, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $12, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $12, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $13, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $13, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $14, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $14, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $15, %xmm1, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $15, %eax, %xmm2, %xmm1 -; AVX2-NEXT: vpextrb $1, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpextrb $0, %xmm0, %ecx -; AVX2-NEXT: bsfl %ecx, %ecx -; AVX2-NEXT: vmovd %ecx, %xmm2 -; AVX2-NEXT: vpinsrb $1, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $2, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $2, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $3, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $3, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $4, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $4, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $5, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $5, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $6, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $6, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $7, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $7, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $8, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $8, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $9, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $9, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $10, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $10, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $11, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $11, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $12, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $12, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $13, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $13, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $14, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $14, %eax, %xmm2, %xmm2 -; AVX2-NEXT: vpextrb $15, %xmm0, %eax -; AVX2-NEXT: bsfl %eax, %eax -; AVX2-NEXT: vpinsrb $15, %eax, %xmm2, %xmm0 -; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-NEXT: vpxor %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vpsubb %ymm0, %ymm1, %ymm1 +; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpsubb {{.*}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm2 +; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm2 +; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0 +; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpshufb %ymm0, %ymm3, %ymm0 +; AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0 ; AVX2-NEXT: retq %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %in, i1 -1) ret <32 x i8> %out } -define <4 x i64> @foldv4i64() { -; AVX-LABEL: foldv4i64: -; AVX: # BB#0: -; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,64,0] -; AVX-NEXT: retq +define <4 x i64> @foldv4i64() nounwind { +; ALL-LABEL: foldv4i64: +; ALL: # BB#0: +; ALL-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,64,0] +; ALL-NEXT: retq %out = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> , i1 0) ret <4 x i64> %out } -define <4 x i64> @foldv4i64u() { -; AVX-LABEL: foldv4i64u: -; AVX: # BB#0: -; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,64,0] -; AVX-NEXT: retq +define <4 x i64> @foldv4i64u() nounwind { +; ALL-LABEL: foldv4i64u: +; ALL: # BB#0: +; ALL-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,64,0] +; ALL-NEXT: retq %out = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> , i1 -1) ret <4 x i64> %out } -define <8 x i32> @foldv8i32() { -; AVX-LABEL: foldv8i32: -; AVX: # BB#0: -; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,32,0,16,0,3,3] -; AVX-NEXT: retq +define <8 x i32> @foldv8i32() nounwind { +; ALL-LABEL: foldv8i32: +; ALL: # BB#0: +; ALL-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,32,0,16,0,3,3] +; ALL-NEXT: retq %out = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> , i1 0) ret <8 x i32> %out } -define <8 x i32> @foldv8i32u() { -; AVX-LABEL: foldv8i32u: -; AVX: # BB#0: -; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,32,0,16,0,3,3] -; AVX-NEXT: retq +define <8 x i32> @foldv8i32u() nounwind { +; ALL-LABEL: foldv8i32u: +; ALL: # BB#0: +; ALL-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,32,0,16,0,3,3] +; ALL-NEXT: retq %out = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> , i1 -1) ret <8 x i32> %out } -define <16 x i16> @foldv16i16() { -; AVX-LABEL: foldv16i16: -; AVX: # BB#0: -; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,16,0,16,0,3,3,1,1,0,1,2,3,4,5] -; AVX-NEXT: retq +define <16 x i16> @foldv16i16() nounwind { +; ALL-LABEL: foldv16i16: +; ALL: # BB#0: +; ALL-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,16,0,16,0,3,3,1,1,0,1,2,3,4,5] +; ALL-NEXT: retq %out = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> , i1 0) ret <16 x i16> %out } -define <16 x i16> @foldv16i16u() { -; AVX-LABEL: foldv16i16u: -; AVX: # BB#0: -; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,16,0,16,0,3,3,1,1,0,1,2,3,4,5] -; AVX-NEXT: retq +define <16 x i16> @foldv16i16u() nounwind { +; ALL-LABEL: foldv16i16u: +; ALL: # BB#0: +; ALL-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,16,0,16,0,3,3,1,1,0,1,2,3,4,5] +; ALL-NEXT: retq %out = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> , i1 -1) ret <16 x i16> %out } -define <32 x i8> @foldv32i8() { -; AVX-LABEL: foldv32i8: -; AVX: # BB#0: -; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5,6,7,8,8,7,6,5,4,3,2,1,0,0,0,0,0] -; AVX-NEXT: retq +define <32 x i8> @foldv32i8() nounwind { +; ALL-LABEL: foldv32i8: +; ALL: # BB#0: +; ALL-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5,6,7,8,8,7,6,5,4,3,2,1,0,0,0,0,0] +; ALL-NEXT: retq %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> , i1 0) ret <32 x i8> %out } -define <32 x i8> @foldv32i8u() { -; AVX-LABEL: foldv32i8u: -; AVX: # BB#0: -; AVX-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5,6,7,8,8,7,6,5,4,3,2,1,0,0,0,0,0] -; AVX-NEXT: retq +define <32 x i8> @foldv32i8u() nounwind { +; ALL-LABEL: foldv32i8u: +; ALL: # BB#0: +; ALL-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,0,3,3,1,1,0,1,2,3,4,5,6,7,8,8,7,6,5,4,3,2,1,0,0,0,0,0] +; ALL-NEXT: retq %out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> , i1 -1) ret <32 x i8> %out } diff --git a/test/CodeGen/X86/vector-tzcnt-512.ll b/test/CodeGen/X86/vector-tzcnt-512.ll new file mode 100644 index 000000000000..9265fad0176c --- /dev/null +++ b/test/CodeGen/X86/vector-tzcnt-512.ll @@ -0,0 +1,271 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512cd | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512CD + +define <8 x i64> @testv8i64(<8 x i64> %in) nounwind { +; ALL-LABEL: testv8i64: +; ALL: ## BB#0: +; ALL-NEXT: vextracti32x4 $3, %zmm0, %xmm1 +; ALL-NEXT: vpextrq $1, %xmm1, %rax +; ALL-NEXT: tzcntq %rax, %rax +; ALL-NEXT: vmovq %rax, %xmm2 +; ALL-NEXT: vmovq %xmm1, %rax +; ALL-NEXT: tzcntq %rax, %rax +; ALL-NEXT: vmovq %rax, %xmm1 +; ALL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] +; ALL-NEXT: vextracti32x4 $2, %zmm0, %xmm2 +; ALL-NEXT: vpextrq $1, %xmm2, %rax +; ALL-NEXT: tzcntq %rax, %rax +; ALL-NEXT: vmovq %rax, %xmm3 +; ALL-NEXT: vmovq %xmm2, %rax +; ALL-NEXT: tzcntq %rax, %rax +; ALL-NEXT: vmovq %rax, %xmm2 +; ALL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0] +; ALL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1 +; ALL-NEXT: vextracti32x4 $1, %zmm0, %xmm2 +; ALL-NEXT: vpextrq $1, %xmm2, %rax +; ALL-NEXT: tzcntq %rax, %rax +; ALL-NEXT: vmovq %rax, %xmm3 +; ALL-NEXT: vmovq %xmm2, %rax +; ALL-NEXT: tzcntq %rax, %rax +; ALL-NEXT: vmovq %rax, %xmm2 +; ALL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0] +; ALL-NEXT: vpextrq $1, %xmm0, %rax +; ALL-NEXT: tzcntq %rax, %rax +; ALL-NEXT: vmovq %rax, %xmm3 +; ALL-NEXT: vmovq %xmm0, %rax +; ALL-NEXT: tzcntq %rax, %rax +; ALL-NEXT: vmovq %rax, %xmm0 +; ALL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0] +; ALL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; ALL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 +; ALL-NEXT: retq + %out = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %in, i1 0) + ret <8 x i64> %out +} + +define <8 x i64> @testv8i64u(<8 x i64> %in) nounwind { +; ALL-LABEL: testv8i64u: +; ALL: ## BB#0: +; ALL-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; ALL-NEXT: vpsubq %zmm0, %zmm1, %zmm1 +; ALL-NEXT: vpandq %zmm1, %zmm0, %zmm0 +; ALL-NEXT: vplzcntq %zmm0, %zmm0 +; ALL-NEXT: vpbroadcastq {{.*}}(%rip), %zmm1 +; ALL-NEXT: vpsubq %zmm0, %zmm1, %zmm0 +; ALL-NEXT: retq + %out = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %in, i1 -1) + ret <8 x i64> %out +} + +define <16 x i32> @testv16i32(<16 x i32> %in) nounwind { +; ALL-LABEL: testv16i32: +; ALL: ## BB#0: +; ALL-NEXT: vextracti32x4 $3, %zmm0, %xmm1 +; ALL-NEXT: vpextrd $1, %xmm1, %eax +; ALL-NEXT: tzcntl %eax, %eax +; ALL-NEXT: vmovd %xmm1, %ecx +; ALL-NEXT: tzcntl %ecx, %ecx +; ALL-NEXT: vmovd %ecx, %xmm2 +; ALL-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2 +; ALL-NEXT: vpextrd $2, %xmm1, %eax +; ALL-NEXT: tzcntl %eax, %eax +; ALL-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2 +; ALL-NEXT: vpextrd $3, %xmm1, %eax +; ALL-NEXT: tzcntl %eax, %eax +; ALL-NEXT: vpinsrd $3, %eax, %xmm2, %xmm1 +; ALL-NEXT: vextracti32x4 $2, %zmm0, %xmm2 +; ALL-NEXT: vpextrd $1, %xmm2, %eax +; ALL-NEXT: tzcntl %eax, %eax +; ALL-NEXT: vmovd %xmm2, %ecx +; ALL-NEXT: tzcntl %ecx, %ecx +; ALL-NEXT: vmovd %ecx, %xmm3 +; ALL-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3 +; ALL-NEXT: vpextrd $2, %xmm2, %eax +; ALL-NEXT: tzcntl %eax, %eax +; ALL-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3 +; ALL-NEXT: vpextrd $3, %xmm2, %eax +; ALL-NEXT: tzcntl %eax, %eax +; ALL-NEXT: vpinsrd $3, %eax, %xmm3, %xmm2 +; ALL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1 +; ALL-NEXT: vextracti32x4 $1, %zmm0, %xmm2 +; ALL-NEXT: vpextrd $1, %xmm2, %eax +; ALL-NEXT: tzcntl %eax, %eax +; ALL-NEXT: vmovd %xmm2, %ecx +; ALL-NEXT: tzcntl %ecx, %ecx +; ALL-NEXT: vmovd %ecx, %xmm3 +; ALL-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3 +; ALL-NEXT: vpextrd $2, %xmm2, %eax +; ALL-NEXT: tzcntl %eax, %eax +; ALL-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3 +; ALL-NEXT: vpextrd $3, %xmm2, %eax +; ALL-NEXT: tzcntl %eax, %eax +; ALL-NEXT: vpinsrd $3, %eax, %xmm3, %xmm2 +; ALL-NEXT: vpextrd $1, %xmm0, %eax +; ALL-NEXT: tzcntl %eax, %eax +; ALL-NEXT: vmovd %xmm0, %ecx +; ALL-NEXT: tzcntl %ecx, %ecx +; ALL-NEXT: vmovd %ecx, %xmm3 +; ALL-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3 +; ALL-NEXT: vpextrd $2, %xmm0, %eax +; ALL-NEXT: tzcntl %eax, %eax +; ALL-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3 +; ALL-NEXT: vpextrd $3, %xmm0, %eax +; ALL-NEXT: tzcntl %eax, %eax +; ALL-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0 +; ALL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 +; ALL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 +; ALL-NEXT: retq + %out = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %in, i1 0) + ret <16 x i32> %out +} + +define <16 x i32> @testv16i32u(<16 x i32> %in) nounwind { +; ALL-LABEL: testv16i32u: +; ALL: ## BB#0: +; ALL-NEXT: vpxord %zmm1, %zmm1, %zmm1 +; ALL-NEXT: vpsubd %zmm0, %zmm1, %zmm1 +; ALL-NEXT: vpandd %zmm1, %zmm0, %zmm0 +; ALL-NEXT: vplzcntd %zmm0, %zmm0 +; ALL-NEXT: vpbroadcastd {{.*}}(%rip), %zmm1 +; ALL-NEXT: vpsubd %zmm0, %zmm1, %zmm0 +; ALL-NEXT: retq + %out = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %in, i1 -1) + ret <16 x i32> %out +} + +define <32 x i16> @testv32i16(<32 x i16> %in) nounwind { +; ALL-LABEL: testv32i16: +; ALL: ## BB#0: +; ALL-NEXT: vpxor %ymm2, %ymm2, %ymm2 +; ALL-NEXT: vpsubw %ymm0, %ymm2, %ymm3 +; ALL-NEXT: vpand %ymm3, %ymm0, %ymm0 +; ALL-NEXT: vmovdqa {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; ALL-NEXT: vpsubw %ymm3, %ymm0, %ymm0 +; ALL-NEXT: vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; ALL-NEXT: vpand %ymm4, %ymm0, %ymm5 +; ALL-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; ALL-NEXT: vpshufb %ymm5, %ymm6, %ymm5 +; ALL-NEXT: vpsrlw $4, %ymm0, %ymm0 +; ALL-NEXT: vpand %ymm4, %ymm0, %ymm0 +; ALL-NEXT: vpshufb %ymm0, %ymm6, %ymm0 +; ALL-NEXT: vpaddb %ymm5, %ymm0, %ymm0 +; ALL-NEXT: vpsllw $8, %ymm0, %ymm5 +; ALL-NEXT: vpaddb %ymm0, %ymm5, %ymm0 +; ALL-NEXT: vpsrlw $8, %ymm0, %ymm0 +; ALL-NEXT: vpsubw %ymm1, %ymm2, %ymm2 +; ALL-NEXT: vpand %ymm2, %ymm1, %ymm1 +; ALL-NEXT: vpsubw %ymm3, %ymm1, %ymm1 +; ALL-NEXT: vpand %ymm4, %ymm1, %ymm2 +; ALL-NEXT: vpshufb %ymm2, %ymm6, %ymm2 +; ALL-NEXT: vpsrlw $4, %ymm1, %ymm1 +; ALL-NEXT: vpand %ymm4, %ymm1, %ymm1 +; ALL-NEXT: vpshufb %ymm1, %ymm6, %ymm1 +; ALL-NEXT: vpaddb %ymm2, %ymm1, %ymm1 +; ALL-NEXT: vpsllw $8, %ymm1, %ymm2 +; ALL-NEXT: vpaddb %ymm1, %ymm2, %ymm1 +; ALL-NEXT: vpsrlw $8, %ymm1, %ymm1 +; ALL-NEXT: retq + %out = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %in, i1 0) + ret <32 x i16> %out +} + +define <32 x i16> @testv32i16u(<32 x i16> %in) nounwind { +; ALL-LABEL: testv32i16u: +; ALL: ## BB#0: +; ALL-NEXT: vpxor %ymm2, %ymm2, %ymm2 +; ALL-NEXT: vpsubw %ymm0, %ymm2, %ymm3 +; ALL-NEXT: vpand %ymm3, %ymm0, %ymm0 +; ALL-NEXT: vmovdqa {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; ALL-NEXT: vpsubw %ymm3, %ymm0, %ymm0 +; ALL-NEXT: vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; ALL-NEXT: vpand %ymm4, %ymm0, %ymm5 +; ALL-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; ALL-NEXT: vpshufb %ymm5, %ymm6, %ymm5 +; ALL-NEXT: vpsrlw $4, %ymm0, %ymm0 +; ALL-NEXT: vpand %ymm4, %ymm0, %ymm0 +; ALL-NEXT: vpshufb %ymm0, %ymm6, %ymm0 +; ALL-NEXT: vpaddb %ymm5, %ymm0, %ymm0 +; ALL-NEXT: vpsllw $8, %ymm0, %ymm5 +; ALL-NEXT: vpaddb %ymm0, %ymm5, %ymm0 +; ALL-NEXT: vpsrlw $8, %ymm0, %ymm0 +; ALL-NEXT: vpsubw %ymm1, %ymm2, %ymm2 +; ALL-NEXT: vpand %ymm2, %ymm1, %ymm1 +; ALL-NEXT: vpsubw %ymm3, %ymm1, %ymm1 +; ALL-NEXT: vpand %ymm4, %ymm1, %ymm2 +; ALL-NEXT: vpshufb %ymm2, %ymm6, %ymm2 +; ALL-NEXT: vpsrlw $4, %ymm1, %ymm1 +; ALL-NEXT: vpand %ymm4, %ymm1, %ymm1 +; ALL-NEXT: vpshufb %ymm1, %ymm6, %ymm1 +; ALL-NEXT: vpaddb %ymm2, %ymm1, %ymm1 +; ALL-NEXT: vpsllw $8, %ymm1, %ymm2 +; ALL-NEXT: vpaddb %ymm1, %ymm2, %ymm1 +; ALL-NEXT: vpsrlw $8, %ymm1, %ymm1 +; ALL-NEXT: retq + %out = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %in, i1 -1) + ret <32 x i16> %out +} + +define <64 x i8> @testv64i8(<64 x i8> %in) nounwind { +; ALL-LABEL: testv64i8: +; ALL: ## BB#0: +; ALL-NEXT: vpxor %ymm2, %ymm2, %ymm2 +; ALL-NEXT: vpsubb %ymm0, %ymm2, %ymm3 +; ALL-NEXT: vpand %ymm3, %ymm0, %ymm0 +; ALL-NEXT: vmovdqa {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; ALL-NEXT: vpsubb %ymm3, %ymm0, %ymm0 +; ALL-NEXT: vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; ALL-NEXT: vpand %ymm4, %ymm0, %ymm5 +; ALL-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; ALL-NEXT: vpshufb %ymm5, %ymm6, %ymm5 +; ALL-NEXT: vpsrlw $4, %ymm0, %ymm0 +; ALL-NEXT: vpand %ymm4, %ymm0, %ymm0 +; ALL-NEXT: vpshufb %ymm0, %ymm6, %ymm0 +; ALL-NEXT: vpaddb %ymm5, %ymm0, %ymm0 +; ALL-NEXT: vpsubb %ymm1, %ymm2, %ymm2 +; ALL-NEXT: vpand %ymm2, %ymm1, %ymm1 +; ALL-NEXT: vpsubb %ymm3, %ymm1, %ymm1 +; ALL-NEXT: vpand %ymm4, %ymm1, %ymm2 +; ALL-NEXT: vpshufb %ymm2, %ymm6, %ymm2 +; ALL-NEXT: vpsrlw $4, %ymm1, %ymm1 +; ALL-NEXT: vpand %ymm4, %ymm1, %ymm1 +; ALL-NEXT: vpshufb %ymm1, %ymm6, %ymm1 +; ALL-NEXT: vpaddb %ymm2, %ymm1, %ymm1 +; ALL-NEXT: retq + %out = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %in, i1 0) + ret <64 x i8> %out +} + +define <64 x i8> @testv64i8u(<64 x i8> %in) nounwind { +; ALL-LABEL: testv64i8u: +; ALL: ## BB#0: +; ALL-NEXT: vpxor %ymm2, %ymm2, %ymm2 +; ALL-NEXT: vpsubb %ymm0, %ymm2, %ymm3 +; ALL-NEXT: vpand %ymm3, %ymm0, %ymm0 +; ALL-NEXT: vmovdqa {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; ALL-NEXT: vpsubb %ymm3, %ymm0, %ymm0 +; ALL-NEXT: vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; ALL-NEXT: vpand %ymm4, %ymm0, %ymm5 +; ALL-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; ALL-NEXT: vpshufb %ymm5, %ymm6, %ymm5 +; ALL-NEXT: vpsrlw $4, %ymm0, %ymm0 +; ALL-NEXT: vpand %ymm4, %ymm0, %ymm0 +; ALL-NEXT: vpshufb %ymm0, %ymm6, %ymm0 +; ALL-NEXT: vpaddb %ymm5, %ymm0, %ymm0 +; ALL-NEXT: vpsubb %ymm1, %ymm2, %ymm2 +; ALL-NEXT: vpand %ymm2, %ymm1, %ymm1 +; ALL-NEXT: vpsubb %ymm3, %ymm1, %ymm1 +; ALL-NEXT: vpand %ymm4, %ymm1, %ymm2 +; ALL-NEXT: vpshufb %ymm2, %ymm6, %ymm2 +; ALL-NEXT: vpsrlw $4, %ymm1, %ymm1 +; ALL-NEXT: vpand %ymm4, %ymm1, %ymm1 +; ALL-NEXT: vpshufb %ymm1, %ymm6, %ymm1 +; ALL-NEXT: vpaddb %ymm2, %ymm1, %ymm1 +; ALL-NEXT: retq + %out = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %in, i1 -1) + ret <64 x i8> %out +} + +declare <8 x i64> @llvm.cttz.v8i64(<8 x i64>, i1) +declare <16 x i32> @llvm.cttz.v16i32(<16 x i32>, i1) +declare <32 x i16> @llvm.cttz.v32i16(<32 x i16>, i1) +declare <64 x i8> @llvm.cttz.v64i8(<64 x i8>, i1) diff --git a/test/CodeGen/X86/vector-zext.ll b/test/CodeGen/X86/vector-zext.ll index b119f5eb89f6..b8024203ab2f 100644 --- a/test/CodeGen/X86/vector-zext.ll +++ b/test/CodeGen/X86/vector-zext.ll @@ -1,332 +1,270 @@ -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=SSE --check-prefix=SSSE3 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2 +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefix=SSE --check-prefix=SSSE3 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2 -define <8 x i32> @zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp { -; SSE2-LABEL: zext_8i16_to_8i32: +define <8 x i16> @zext_16i8_to_8i16(<16 x i8> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: zext_16i8_to_8i16: ; SSE2: # BB#0: # %entry -; SSE2-NEXT: movdqa %xmm0, %xmm1 -; SSE2-NEXT: pxor %xmm2, %xmm2 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] -; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] -; SSE2-NEXT: pand {{.*}}(%rip), %xmm1 +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] ; SSE2-NEXT: retq ; -; SSSE3-LABEL: zext_8i16_to_8i32: +; SSSE3-LABEL: zext_16i8_to_8i16: ; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: movdqa %xmm0, %xmm1 -; SSSE3-NEXT: pxor %xmm2, %xmm2 -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] -; SSSE3-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] -; SSSE3-NEXT: pand {{.*}}(%rip), %xmm1 +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] ; SSSE3-NEXT: retq ; -; SSE41-LABEL: zext_8i16_to_8i32: +; SSE41-LABEL: zext_16i8_to_8i16: ; SSE41: # BB#0: # %entry -; SSE41-NEXT: movdqa %xmm0, %xmm1 -; SSE41-NEXT: pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero -; SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] -; SSE41-NEXT: pand {{.*}}(%rip), %xmm1 +; SSE41-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero ; SSE41-NEXT: retq ; -; AVX1-LABEL: zext_8i16_to_8i32: -; AVX1: # BB#0: # %entry -; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] -; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; AVX1-NEXT: retq -; -; AVX2-LABEL: zext_8i16_to_8i32: -; AVX2: # BB#0: # %entry -; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero -; AVX2-NEXT: retq +; AVX-LABEL: zext_16i8_to_8i16: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX-NEXT: retq entry: - %B = zext <8 x i16> %A to <8 x i32> - ret <8 x i32>%B + %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> + %C = zext <8 x i8> %B to <8 x i16> + ret <8 x i16> %C } -define <4 x i64> @zext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp { -; SSE2-LABEL: zext_4i32_to_4i64: +; PR17654 +define <16 x i16> @zext_16i8_to_16i16(<16 x i8> %A) { +; SSE2-LABEL: zext_16i8_to_16i16: ; SSE2: # BB#0: # %entry -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,1,1,3] -; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [4294967295,4294967295] -; SSE2-NEXT: pand %xmm3, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3] -; SSE2-NEXT: pand %xmm3, %xmm1 -; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15] ; SSE2-NEXT: retq ; -; SSSE3-LABEL: zext_4i32_to_4i64: +; SSSE3-LABEL: zext_16i8_to_16i16: ; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,1,1,3] -; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [4294967295,4294967295] -; SSSE3-NEXT: pand %xmm3, %xmm2 -; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3] -; SSSE3-NEXT: pand %xmm3, %xmm1 -; SSSE3-NEXT: movdqa %xmm2, %xmm0 +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; SSSE3-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15] ; SSSE3-NEXT: retq ; -; SSE41-LABEL: zext_4i32_to_4i64: +; SSE41-LABEL: zext_16i8_to_16i16: ; SSE41: # BB#0: # %entry -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero -; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [4294967295,4294967295] -; SSE41-NEXT: pand %xmm3, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3] -; SSE41-NEXT: pand %xmm3, %xmm1 -; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero +; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15] ; SSE41-NEXT: retq ; -; AVX1-LABEL: zext_4i32_to_4i64: +; AVX1-LABEL: zext_16i8_to_16i16: ; AVX1: # BB#0: # %entry ; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm1 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] -; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero +; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; -; AVX2-LABEL: zext_4i32_to_4i64: +; AVX2-LABEL: zext_16i8_to_16i16: ; AVX2: # BB#0: # %entry -; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero ; AVX2-NEXT: retq entry: - %B = zext <4 x i32> %A to <4 x i64> - ret <4 x i64>%B + %B = zext <16 x i8> %A to <16 x i16> + ret <16 x i16> %B } -define <8 x i32> @zext_8i8_to_8i32(<8 x i8> %z) { -; SSE2-LABEL: zext_8i8_to_8i32: +define <4 x i32> @zext_16i8_to_4i32(<16 x i8> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: zext_16i8_to_4i32: ; SSE2: # BB#0: # %entry -; SSE2-NEXT: movdqa %xmm0, %xmm2 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] -; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255] -; SSE2-NEXT: pand %xmm1, %xmm2 -; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7] -; SSE2-NEXT: pand %xmm0, %xmm1 -; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] ; SSE2-NEXT: retq ; -; SSSE3-LABEL: zext_8i8_to_8i32: +; SSSE3-LABEL: zext_16i8_to_4i32: ; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: movdqa %xmm0, %xmm2 -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] -; SSSE3-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255] -; SSSE3-NEXT: pand %xmm1, %xmm2 -; SSSE3-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7] -; SSSE3-NEXT: pand %xmm0, %xmm1 -; SSSE3-NEXT: movdqa %xmm2, %xmm0 +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] ; SSSE3-NEXT: retq ; -; SSE41-LABEL: zext_8i8_to_8i32: +; SSE41-LABEL: zext_16i8_to_4i32: ; SSE41: # BB#0: # %entry -; SSE41-NEXT: pmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero -; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255] -; SSE41-NEXT: pand %xmm1, %xmm2 -; SSE41-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7] -; SSE41-NEXT: pand %xmm0, %xmm1 -; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero ; SSE41-NEXT: retq ; -; AVX1-LABEL: zext_8i8_to_8i32: -; AVX1: # BB#0: # %entry -; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero -; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7] -; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 -; AVX1-NEXT: vandps {{.*}}(%rip), %ymm0, %ymm0 -; AVX1-NEXT: retq -; -; AVX2-LABEL: zext_8i8_to_8i32: -; AVX2: # BB#0: # %entry -; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero -; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %ymm1 -; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: retq +; AVX-LABEL: zext_16i8_to_4i32: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX-NEXT: retq entry: - %t = zext <8 x i8> %z to <8 x i32> - ret <8 x i32> %t + %B = shufflevector <16 x i8> %A, <16 x i8> undef, <4 x i32> + %C = zext <4 x i8> %B to <4 x i32> + ret <4 x i32> %C } -; PR17654 -define <16 x i16> @zext_16i8_to_16i16(<16 x i8> %z) { -; SSE2-LABEL: zext_16i8_to_16i16: +define <8 x i32> @zext_16i8_to_8i32(<16 x i8> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: zext_16i8_to_8i32: ; SSE2: # BB#0: # %entry ; SSE2-NEXT: movdqa %xmm0, %xmm1 ; SSE2-NEXT: pxor %xmm2, %xmm2 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] -; SSE2-NEXT: pand {{.*}}(%rip), %xmm1 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] ; SSE2-NEXT: retq ; -; SSSE3-LABEL: zext_16i8_to_16i16: +; SSSE3-LABEL: zext_16i8_to_8i32: ; SSSE3: # BB#0: # %entry ; SSSE3-NEXT: movdqa %xmm0, %xmm1 ; SSSE3-NEXT: pxor %xmm2, %xmm2 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; SSSE3-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] -; SSSE3-NEXT: pand {{.*}}(%rip), %xmm1 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSSE3-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] ; SSSE3-NEXT: retq ; -; SSE41-LABEL: zext_16i8_to_16i16: +; SSE41-LABEL: zext_16i8_to_8i32: ; SSE41: # BB#0: # %entry -; SSE41-NEXT: movdqa %xmm0, %xmm1 -; SSE41-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero -; SSE41-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] -; SSE41-NEXT: pand {{.*}}(%rip), %xmm1 +; SSE41-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; SSE41-NEXT: pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; SSE41-NEXT: movdqa %xmm2, %xmm0 ; SSE41-NEXT: retq ; -; AVX1-LABEL: zext_16i8_to_16i16: +; AVX1-LABEL: zext_16i8_to_8i32: ; AVX1: # BB#0: # %entry -; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] -; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0] +; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; -; AVX2-LABEL: zext_16i8_to_16i16: +; AVX2-LABEL: zext_16i8_to_8i32: ; AVX2: # BB#0: # %entry -; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 ; AVX2-NEXT: retq entry: - %t = zext <16 x i8> %z to <16 x i16> - ret <16 x i16> %t + %B = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> + %C = zext <8 x i8> %B to <8 x i32> + ret <8 x i32> %C } -define <16 x i16> @load_zext_16i8_to_16i16(<16 x i8> *%ptr) { -; SSE2-LABEL: load_zext_16i8_to_16i16: +define <2 x i64> @zext_16i8_to_2i64(<16 x i8> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: zext_16i8_to_2i64: ; SSE2: # BB#0: # %entry -; SSE2-NEXT: movdqa (%rdi), %xmm1 -; SSE2-NEXT: pxor %xmm2, %xmm2 -; SSE2-NEXT: movdqa %xmm1, %xmm0 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] -; SSE2-NEXT: pand {{.*}}(%rip), %xmm1 +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; SSE2-NEXT: retq ; -; SSSE3-LABEL: load_zext_16i8_to_16i16: +; SSSE3-LABEL: zext_16i8_to_2i64: ; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: movdqa (%rdi), %xmm1 -; SSSE3-NEXT: pxor %xmm2, %xmm2 -; SSSE3-NEXT: movdqa %xmm1, %xmm0 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; SSSE3-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] -; SSSE3-NEXT: pand {{.*}}(%rip), %xmm1 +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero ; SSSE3-NEXT: retq ; -; SSE41-LABEL: load_zext_16i8_to_16i16: +; SSE41-LABEL: zext_16i8_to_2i64: ; SSE41: # BB#0: # %entry -; SSE41-NEXT: pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero -; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero +; SSE41-NEXT: pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero ; SSE41-NEXT: retq ; -; AVX1-LABEL: load_zext_16i8_to_16i16: -; AVX1: # BB#0: # %entry -; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero -; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; AVX1-NEXT: retq -; -; AVX2-LABEL: load_zext_16i8_to_16i16: -; AVX2: # BB#0: # %entry -; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero -; AVX2-NEXT: retq +; AVX-LABEL: zext_16i8_to_2i64: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; AVX-NEXT: retq entry: - %X = load <16 x i8>, <16 x i8>* %ptr - %Y = zext <16 x i8> %X to <16 x i16> - ret <16 x i16> %Y + %B = shufflevector <16 x i8> %A, <16 x i8> undef, <2 x i32> + %C = zext <2 x i8> %B to <2 x i64> + ret <2 x i64> %C } -define <8 x i32> @load_zext_8i16_to_8i32(<8 x i16> *%ptr) { -; SSE2-LABEL: load_zext_8i16_to_8i32: +define <4 x i64> @zext_16i8_to_4i64(<16 x i8> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: zext_16i8_to_4i64: ; SSE2: # BB#0: # %entry -; SSE2-NEXT: movdqa (%rdi), %xmm1 +; SSE2-NEXT: movdqa %xmm0, %xmm1 ; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] ; SSE2-NEXT: movdqa %xmm1, %xmm0 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] -; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] -; SSE2-NEXT: pand {{.*}}(%rip), %xmm1 +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3] ; SSE2-NEXT: retq ; -; SSSE3-LABEL: load_zext_8i16_to_8i32: +; SSSE3-LABEL: zext_16i8_to_4i64: ; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: movdqa (%rdi), %xmm1 -; SSSE3-NEXT: pxor %xmm2, %xmm2 -; SSSE3-NEXT: movdqa %xmm1, %xmm0 -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] -; SSSE3-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7] -; SSSE3-NEXT: pand {{.*}}(%rip), %xmm1 +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; SSSE3-NEXT: pshufb {{.*#+}} xmm1 = xmm1[2],zero,zero,zero,zero,zero,zero,zero,xmm1[3],zero,zero,zero,zero,zero,zero,zero ; SSSE3-NEXT: retq ; -; SSE41-LABEL: load_zext_8i16_to_8i32: +; SSE41-LABEL: zext_16i8_to_4i64: ; SSE41: # BB#0: # %entry -; SSE41-NEXT: pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero -; SSE41-NEXT: pmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero +; SSE41-NEXT: pmovzxbq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; SSE41-NEXT: psrld $16, %xmm0 +; SSE41-NEXT: pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; SSE41-NEXT: movdqa %xmm2, %xmm0 ; SSE41-NEXT: retq ; -; AVX1-LABEL: load_zext_8i16_to_8i32: +; AVX1-LABEL: zext_16i8_to_4i64: ; AVX1: # BB#0: # %entry -; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero -; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3] +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0] +; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; -; AVX2-LABEL: load_zext_8i16_to_8i32: +; AVX2-LABEL: zext_16i8_to_4i64: ; AVX2: # BB#0: # %entry -; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero +; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero +; AVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 ; AVX2-NEXT: retq entry: - %X = load <8 x i16>, <8 x i16>* %ptr - %Y = zext <8 x i16> %X to <8 x i32> - ret <8 x i32>%Y + %B = shufflevector <16 x i8> %A, <16 x i8> undef, <4 x i32> + %C = zext <4 x i8> %B to <4 x i64> + ret <4 x i64> %C } -define <4 x i64> @load_zext_4i32_to_4i64(<4 x i32> *%ptr) { -; SSE2-LABEL: load_zext_4i32_to_4i64: +define <4 x i32> @zext_8i16_to_4i32(<8 x i16> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: zext_8i16_to_4i32: ; SSE2: # BB#0: # %entry -; SSE2-NEXT: movdqa (%rdi), %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,1,1,3] -; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [4294967295,4294967295] -; SSE2-NEXT: pand %xmm2, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3] -; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] ; SSE2-NEXT: retq ; -; SSSE3-LABEL: load_zext_4i32_to_4i64: +; SSSE3-LABEL: zext_8i16_to_4i32: ; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: movdqa (%rdi), %xmm1 -; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,1,1,3] -; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [4294967295,4294967295] -; SSSE3-NEXT: pand %xmm2, %xmm0 -; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3] -; SSSE3-NEXT: pand %xmm2, %xmm1 +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] ; SSSE3-NEXT: retq ; -; SSE41-LABEL: load_zext_4i32_to_4i64: +; SSE41-LABEL: zext_8i16_to_4i32: ; SSE41: # BB#0: # %entry -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero ; SSE41-NEXT: retq ; -; AVX1-LABEL: load_zext_4i32_to_4i64: -; AVX1: # BB#0: # %entry -; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero -; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; AVX1-NEXT: retq -; -; AVX2-LABEL: load_zext_4i32_to_4i64: -; AVX2: # BB#0: # %entry -; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero -; AVX2-NEXT: retq +; AVX-LABEL: zext_8i16_to_4i32: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX-NEXT: retq entry: - %X = load <4 x i32>, <4 x i32>* %ptr - %Y = zext <4 x i32> %X to <4 x i64> - ret <4 x i64>%Y + %B = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> + %C = zext <4 x i16> %B to <4 x i32> + ret <4 x i32> %C } -define <8 x i32> @shuf_zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp { -; SSE2-LABEL: shuf_zext_8i16_to_8i32: +define <8 x i32> @zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: zext_8i16_to_8i32: ; SSE2: # BB#0: # %entry ; SSE2-NEXT: movdqa %xmm0, %xmm1 ; SSE2-NEXT: pxor %xmm2, %xmm2 @@ -334,7 +272,7 @@ define <8 x i32> @shuf_zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] ; SSE2-NEXT: retq ; -; SSSE3-LABEL: shuf_zext_8i16_to_8i32: +; SSSE3-LABEL: zext_8i16_to_8i32: ; SSSE3: # BB#0: # %entry ; SSSE3-NEXT: movdqa %xmm0, %xmm1 ; SSSE3-NEXT: pxor %xmm2, %xmm2 @@ -342,7 +280,7 @@ define <8 x i32> @shuf_zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ; SSSE3-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] ; SSSE3-NEXT: retq ; -; SSE41-LABEL: shuf_zext_8i16_to_8i32: +; SSE41-LABEL: zext_8i16_to_8i32: ; SSE41: # BB#0: # %entry ; SSE41-NEXT: movdqa %xmm0, %xmm1 ; SSE41-NEXT: pxor %xmm2, %xmm2 @@ -350,7 +288,7 @@ define <8 x i32> @shuf_zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ; SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] ; SSE41-NEXT: retq ; -; AVX1-LABEL: shuf_zext_8i16_to_8i32: +; AVX1-LABEL: zext_8i16_to_8i32: ; AVX1: # BB#0: # %entry ; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] @@ -358,101 +296,791 @@ define <8 x i32> @shuf_zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; -; AVX2-LABEL: shuf_zext_8i16_to_8i32: +; AVX2-LABEL: zext_8i16_to_8i32: ; AVX2: # BB#0: # %entry ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero ; AVX2-NEXT: retq entry: - %B = shufflevector <8 x i16> %A, <8 x i16> zeroinitializer, <16 x i32> - %Z = bitcast <16 x i16> %B to <8 x i32> - ret <8 x i32> %Z + %B = zext <8 x i16> %A to <8 x i32> + ret <8 x i32>%B } -define <4 x i64> @shuf_zext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp { -; SSE2-LABEL: shuf_zext_4i32_to_4i64: +define <2 x i64> @zext_8i16_to_2i64(<8 x i16> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: zext_8i16_to_2i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: zext_8i16_to_2i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: zext_8i16_to_2i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; SSE41-NEXT: retq +; +; AVX-LABEL: zext_8i16_to_2i64: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; AVX-NEXT: retq +entry: + %B = shufflevector <8 x i16> %A, <8 x i16> undef, <2 x i32> + %C = zext <2 x i16> %B to <2 x i64> + ret <2 x i64> %C +} + +define <4 x i64> @zext_8i16_to_4i64(<8 x i16> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: zext_8i16_to_4i64: ; SSE2: # BB#0: # %entry ; SSE2-NEXT: movdqa %xmm0, %xmm1 ; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSE2-NEXT: movdqa %xmm1, %xmm0 ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] ; SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3] ; SSE2-NEXT: retq ; -; SSSE3-LABEL: shuf_zext_4i32_to_4i64: +; SSSE3-LABEL: zext_8i16_to_4i64: ; SSSE3: # BB#0: # %entry ; SSSE3-NEXT: movdqa %xmm0, %xmm1 ; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSSE3-NEXT: movdqa %xmm1, %xmm0 ; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] ; SSSE3-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3] ; SSSE3-NEXT: retq ; -; SSE41-LABEL: shuf_zext_4i32_to_4i64: +; SSE41-LABEL: zext_8i16_to_4i64: ; SSE41: # BB#0: # %entry -; SSE41-NEXT: movdqa %xmm0, %xmm1 -; SSE41-NEXT: pxor %xmm2, %xmm2 -; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero -; SSE41-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSE41-NEXT: pmovzxwq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; SSE41-NEXT: pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; SSE41-NEXT: movdqa %xmm2, %xmm0 ; SSE41-NEXT: retq ; -; AVX1-LABEL: shuf_zext_4i32_to_4i64: +; AVX1-LABEL: zext_8i16_to_4i64: ; AVX1: # BB#0: # %entry -; AVX1-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero -; AVX1-NEXT: vxorpd %xmm2, %xmm2, %xmm2 -; AVX1-NEXT: vblendpd {{.*#+}} xmm0 = xmm2[0],xmm0[1] -; AVX1-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[2,0,3,0] -; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3] +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7] +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7] +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; -; AVX2-LABEL: shuf_zext_4i32_to_4i64: +; AVX2-LABEL: zext_8i16_to_4i64: ; AVX2: # BB#0: # %entry -; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX2-NEXT: vpxor %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15] ; AVX2-NEXT: retq entry: - %B = shufflevector <4 x i32> %A, <4 x i32> zeroinitializer, <8 x i32> - %Z = bitcast <8 x i32> %B to <4 x i64> - ret <4 x i64> %Z + %B = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> + %C = zext <4 x i16> %B to <4 x i64> + ret <4 x i64> %C } -define <8 x i32> @shuf_zext_8i8_to_8i32(<8 x i8> %A) { -; SSE2-LABEL: shuf_zext_8i8_to_8i32: +define <2 x i64> @zext_4i32_to_2i64(<4 x i32> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: zext_4i32_to_2i64: ; SSE2: # BB#0: # %entry -; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 -; SSE2-NEXT: packuswb %xmm0, %xmm0 ; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: movdqa %xmm0, %xmm2 -; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] -; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] -; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7] -; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255] -; SSE2-NEXT: pandn %xmm0, %xmm1 -; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; SSE2-NEXT: retq ; -; SSSE3-LABEL: shuf_zext_8i8_to_8i32: +; SSSE3-LABEL: zext_4i32_to_2i64: ; SSSE3: # BB#0: # %entry -; SSSE3-NEXT: movdqa %xmm0, %xmm1 -; SSSE3-NEXT: pshufb {{.*#+}} xmm1 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u] -; SSSE3-NEXT: pxor %xmm2, %xmm2 -; SSSE3-NEXT: movdqa %xmm1, %xmm0 -; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] -; SSSE3-NEXT: pshufb {{.*#+}} xmm1 = xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; SSSE3-NEXT: retq ; -; SSE41-LABEL: shuf_zext_8i8_to_8i32: +; SSE41-LABEL: zext_4i32_to_2i64: ; SSE41: # BB#0: # %entry -; SSE41-NEXT: movdqa %xmm0, %xmm1 -; SSE41-NEXT: pshufb {{.*#+}} xmm1 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u] -; SSE41-NEXT: pmovzxbd {{.*#+}} xmm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero -; SSE41-NEXT: pshufb {{.*#+}} xmm1 = xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero ; SSE41-NEXT: retq ; -; AVX1-LABEL: shuf_zext_8i8_to_8i32: -; AVX1: # BB#0: # %entry -; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u] -; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero -; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero -; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX-LABEL: zext_4i32_to_2i64: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero +; AVX-NEXT: retq +entry: + %B = shufflevector <4 x i32> %A, <4 x i32> undef, <2 x i32> + %C = zext <2 x i32> %B to <2 x i64> + ret <2 x i64> %C +} + +define <4 x i64> @zext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: zext_4i32_to_4i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: zext_4i32_to_4i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSSE3-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: zext_4i32_to_4i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero +; SSE41-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSE41-NEXT: retq +; +; AVX1-LABEL: zext_4i32_to_4i64: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm1 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: zext_4i32_to_4i64: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX2-NEXT: retq +entry: + %B = zext <4 x i32> %A to <4 x i64> + ret <4 x i64>%B +} + +define <2 x i64> @load_zext_2i8_to_2i64(<2 x i8> *%ptr) { +; SSE2-LABEL: load_zext_2i8_to_2i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movzwl (%rdi), %eax +; SSE2-NEXT: movd %eax, %xmm0 +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_zext_2i8_to_2i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movzwl (%rdi), %eax +; SSSE3-NEXT: movd %eax, %xmm0 +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_zext_2i8_to_2i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero +; SSE41-NEXT: retq +; +; AVX-LABEL: load_zext_2i8_to_2i64: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero +; AVX-NEXT: retq +entry: + %X = load <2 x i8>, <2 x i8>* %ptr + %Y = zext <2 x i8> %X to <2 x i64> + ret <2 x i64> %Y +} + +define <4 x i32> @load_zext_4i8_to_4i32(<4 x i8> *%ptr) { +; SSE2-LABEL: load_zext_4i8_to_4i32: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_zext_4i8_to_4i32: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_zext_4i8_to_4i32: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero +; SSE41-NEXT: retq +; +; AVX-LABEL: load_zext_4i8_to_4i32: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero +; AVX-NEXT: retq +entry: + %X = load <4 x i8>, <4 x i8>* %ptr + %Y = zext <4 x i8> %X to <4 x i32> + ret <4 x i32> %Y +} + +define <4 x i64> @load_zext_4i8_to_4i64(<4 x i8> *%ptr) { +; SSE2-LABEL: load_zext_4i8_to_4i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,1,1,3] +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0] +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,3,3] +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_zext_4i8_to_4i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[4],zero,zero,zero,zero,zero,zero,zero +; SSSE3-NEXT: pshufb {{.*#+}} xmm1 = xmm1[8],zero,zero,zero,zero,zero,zero,zero,xmm1[12],zero,zero,zero,zero,zero,zero,zero +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_zext_4i8_to_4i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero +; SSE41-NEXT: pmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero +; SSE41-NEXT: retq +; +; AVX1-LABEL: load_zext_4i8_to_4i64: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: load_zext_4i8_to_4i64: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero +; AVX2-NEXT: retq +entry: + %X = load <4 x i8>, <4 x i8>* %ptr + %Y = zext <4 x i8> %X to <4 x i64> + ret <4 x i64> %Y +} + +define <8 x i16> @load_zext_8i8_to_8i16(<8 x i8> *%ptr) { +; SSE2-LABEL: load_zext_8i8_to_8i16: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_zext_8i8_to_8i16: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_zext_8i8_to_8i16: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero +; SSE41-NEXT: retq +; +; AVX-LABEL: load_zext_8i8_to_8i16: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero +; AVX-NEXT: retq +entry: + %X = load <8 x i8>, <8 x i8>* %ptr + %Y = zext <8 x i8> %X to <8 x i16> + ret <8 x i16> %Y +} + +define <8 x i32> @load_zext_8i8_to_8i32(<8 x i8> *%ptr) { +; SSE2-LABEL: load_zext_8i8_to_8i32: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0] +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_zext_8i8_to_8i32: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movq {{.*#+}} xmm1 = mem[0],zero +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] +; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[6],zero,zero,zero +; SSSE3-NEXT: pshufb {{.*#+}} xmm1 = xmm1[8],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[14],zero,zero,zero +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_zext_8i8_to_8i32: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero +; SSE41-NEXT: pmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero +; SSE41-NEXT: retq +; +; AVX1-LABEL: load_zext_8i8_to_8i32: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: load_zext_8i8_to_8i32: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero +; AVX2-NEXT: retq +entry: + %X = load <8 x i8>, <8 x i8>* %ptr + %Y = zext <8 x i8> %X to <8 x i32> + ret <8 x i32> %Y +} + +define <16 x i16> @load_zext_16i8_to_16i16(<16 x i8> *%ptr) { +; SSE2-LABEL: load_zext_16i8_to_16i16: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa (%rdi), %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_zext_16i8_to_16i16: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movdqa (%rdi), %xmm1 +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; SSSE3-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_zext_16i8_to_16i16: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero +; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero +; SSE41-NEXT: retq +; +; AVX1-LABEL: load_zext_16i8_to_16i16: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: load_zext_16i8_to_16i16: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero +; AVX2-NEXT: retq +entry: + %X = load <16 x i8>, <16 x i8>* %ptr + %Y = zext <16 x i8> %X to <16 x i16> + ret <16 x i16> %Y +} + +define <2 x i64> @load_zext_2i16_to_2i64(<2 x i16> *%ptr) { +; SSE2-LABEL: load_zext_2i16_to_2i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_zext_2i16_to_2i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_zext_2i16_to_2i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero +; SSE41-NEXT: retq +; +; AVX-LABEL: load_zext_2i16_to_2i64: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero +; AVX-NEXT: retq +entry: + %X = load <2 x i16>, <2 x i16>* %ptr + %Y = zext <2 x i16> %X to <2 x i64> + ret <2 x i64> %Y +} + +define <4 x i32> @load_zext_4i16_to_4i32(<4 x i16> *%ptr) { +; SSE2-LABEL: load_zext_4i16_to_4i32: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_zext_4i16_to_4i32: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_zext_4i16_to_4i32: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero +; SSE41-NEXT: retq +; +; AVX-LABEL: load_zext_4i16_to_4i32: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero +; AVX-NEXT: retq +entry: + %X = load <4 x i16>, <4 x i16>* %ptr + %Y = zext <4 x i16> %X to <4 x i32> + ret <4 x i32> %Y +} + +define <4 x i64> @load_zext_4i16_to_4i64(<4 x i16> *%ptr) { +; SSE2-LABEL: load_zext_4i16_to_4i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero +; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,1,1,3] +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [65535,0,0,0,65535,0,0,0] +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,3,3] +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_zext_4i16_to_4i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movq {{.*#+}} xmm1 = mem[0],zero +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,1],zero,zero,zero,zero,zero,zero,xmm0[4,5],zero,zero,zero,zero,zero,zero +; SSSE3-NEXT: pshufb {{.*#+}} xmm1 = xmm1[8,9],zero,zero,zero,zero,zero,zero,xmm1[12,13],zero,zero,zero,zero,zero,zero +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_zext_4i16_to_4i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero +; SSE41-NEXT: pmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero +; SSE41-NEXT: retq +; +; AVX1-LABEL: load_zext_4i16_to_4i64: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: load_zext_4i16_to_4i64: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero +; AVX2-NEXT: retq +entry: + %X = load <4 x i16>, <4 x i16>* %ptr + %Y = zext <4 x i16> %X to <4 x i64> + ret <4 x i64> %Y +} + +define <8 x i32> @load_zext_8i16_to_8i32(<8 x i16> *%ptr) { +; SSE2-LABEL: load_zext_8i16_to_8i32: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa (%rdi), %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_zext_8i16_to_8i32: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movdqa (%rdi), %xmm1 +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSSE3-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_zext_8i16_to_8i32: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero +; SSE41-NEXT: retq +; +; AVX1-LABEL: load_zext_8i16_to_8i32: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: load_zext_8i16_to_8i32: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero +; AVX2-NEXT: retq +entry: + %X = load <8 x i16>, <8 x i16>* %ptr + %Y = zext <8 x i16> %X to <8 x i32> + ret <8 x i32> %Y +} + +define <2 x i64> @load_zext_2i32_to_2i64(<2 x i32> *%ptr) { +; SSE2-LABEL: load_zext_2i32_to_2i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_zext_2i32_to_2i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movq {{.*#+}} xmm0 = mem[0],zero +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_zext_2i32_to_2i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero +; SSE41-NEXT: retq +; +; AVX-LABEL: load_zext_2i32_to_2i64: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero +; AVX-NEXT: retq +entry: + %X = load <2 x i32>, <2 x i32>* %ptr + %Y = zext <2 x i32> %X to <2 x i64> + ret <2 x i64> %Y +} + +define <4 x i64> @load_zext_4i32_to_4i64(<4 x i32> *%ptr) { +; SSE2-LABEL: load_zext_4i32_to_4i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa (%rdi), %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: load_zext_4i32_to_4i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movdqa (%rdi), %xmm1 +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSSE3-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: load_zext_4i32_to_4i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero +; SSE41-NEXT: retq +; +; AVX1-LABEL: load_zext_4i32_to_4i64: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero +; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: load_zext_4i32_to_4i64: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero +; AVX2-NEXT: retq +entry: + %X = load <4 x i32>, <4 x i32>* %ptr + %Y = zext <4 x i32> %X to <4 x i64> + ret <4 x i64> %Y +} + +define <8 x i32> @zext_8i8_to_8i32(<8 x i8> %z) { +; SSE2-LABEL: zext_8i8_to_8i32: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: zext_8i8_to_8i32: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: pand {{.*}}(%rip), %xmm1 +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSSE3-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: zext_8i8_to_8i32: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: pand {{.*}}(%rip), %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero +; SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE41-NEXT: retq +; +; AVX1-LABEL: zext_8i8_to_8i32: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: zext_8i8_to_8i32: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX2-NEXT: retq +entry: + %t = zext <8 x i8> %z to <8 x i32> + ret <8 x i32> %t +} + +define <8 x i32> @shuf_zext_8i16_to_8i32(<8 x i16> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: shuf_zext_8i16_to_8i32: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: shuf_zext_8i16_to_8i32: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSSE3-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: shuf_zext_8i16_to_8i32: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero +; SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE41-NEXT: retq +; +; AVX1-LABEL: shuf_zext_8i16_to_8i32: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuf_zext_8i16_to_8i32: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX2-NEXT: retq +entry: + %B = shufflevector <8 x i16> %A, <8 x i16> zeroinitializer, <16 x i32> + %Z = bitcast <16 x i16> %B to <8 x i32> + ret <8 x i32> %Z +} + +define <4 x i64> @shuf_zext_4i32_to_4i64(<4 x i32> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: shuf_zext_4i32_to_4i64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: shuf_zext_4i32_to_4i64: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSSE3-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: shuf_zext_4i32_to_4i64: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero +; SSE41-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3] +; SSE41-NEXT: retq +; +; AVX1-LABEL: shuf_zext_4i32_to_4i64: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero +; AVX1-NEXT: vxorpd %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vblendpd {{.*#+}} xmm0 = xmm2[0],xmm0[1] +; AVX1-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[2,0,3,0] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuf_zext_4i32_to_4i64: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX2-NEXT: retq +entry: + %B = shufflevector <4 x i32> %A, <4 x i32> zeroinitializer, <8 x i32> + %Z = bitcast <8 x i32> %B to <4 x i64> + ret <4 x i64> %Z +} + +define <8 x i32> @shuf_zext_8i8_to_8i32(<8 x i8> %A) { +; SSE2-LABEL: shuf_zext_8i8_to_8i32: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm1 +; SSE2-NEXT: packuswb %xmm1, %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: shuf_zext_8i8_to_8i32: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: pshufb {{.*#+}} xmm1 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u] +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSSE3-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: shuf_zext_8i8_to_8i32: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u] +; SSE41-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; SSE41-NEXT: pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: retq +; +; AVX1-LABEL: shuf_zext_8i8_to_8i32: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u] +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3] +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: shuf_zext_8i8_to_8i32: @@ -465,3 +1093,346 @@ entry: %Z = bitcast <32 x i8> %B to <8 x i32> ret <8 x i32> %Z } + +define <2 x i64> @shuf_zext_16i8_to_2i64_offset6(<16 x i8> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: shuf_zext_16i8_to_2i64_offset6: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: shuf_zext_16i8_to_2i64_offset6: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[6],zero,zero,zero,zero,zero,zero,zero,xmm0[7],zero,zero,zero,zero,zero,zero,zero +; SSSE3-NEXT: retq +; +; SSE41-LABEL: shuf_zext_16i8_to_2i64_offset6: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: psrlq $48, %xmm0 +; SSE41-NEXT: pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; SSE41-NEXT: retq +; +; AVX-LABEL: shuf_zext_16i8_to_2i64_offset6: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpsrlq $48, %xmm0, %xmm0 +; AVX-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; AVX-NEXT: retq +entry: + %B = shufflevector <16 x i8> %A, <16 x i8> zeroinitializer, <16 x i32> + %Z = bitcast <16 x i8> %B to <2 x i64> + ret <2 x i64> %Z +} + +define <4 x i64> @shuf_zext_16i8_to_4i64_offset11(<16 x i8> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: shuf_zext_16i8_to_4i64_offset11: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: psrldq {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15] +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: shuf_zext_16i8_to_4i64_offset11: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: pshufb {{.*#+}} xmm0 = xmm0[11],zero,zero,zero,zero,zero,zero,zero,xmm0[12],zero,zero,zero,zero,zero,zero,zero +; SSSE3-NEXT: pshufb {{.*#+}} xmm1 = xmm1[13],zero,zero,zero,zero,zero,zero,zero,xmm1[14],zero,zero,zero,zero,zero,zero,zero +; SSSE3-NEXT: retq +; +; SSE41-LABEL: shuf_zext_16i8_to_4i64_offset11: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: psrldq {{.*#+}} xmm1 = xmm1[11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE41-NEXT: pmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero +; SSE41-NEXT: psrldq {{.*#+}} xmm0 = xmm0[13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE41-NEXT: pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: retq +; +; AVX1-LABEL: shuf_zext_16i8_to_4i64_offset11: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpsrldq {{.*#+}} xmm1 = xmm0[11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuf_zext_16i8_to_4i64_offset11: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero +; AVX2-NEXT: retq +entry: + %B = shufflevector <16 x i8> %A, <16 x i8> zeroinitializer, <32 x i32> + %Z = bitcast <32 x i8> %B to <4 x i64> + ret <4 x i64> %Z +} + +define <2 x i64> @shuf_zext_8i16_to_2i64_offset6(<8 x i16> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: shuf_zext_8i16_to_2i64_offset6: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: shuf_zext_8i16_to_2i64_offset6: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSSE3-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: shuf_zext_8i16_to_2i64_offset6: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: psrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero +; SSE41-NEXT: pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; SSE41-NEXT: retq +; +; AVX-LABEL: shuf_zext_8i16_to_2i64_offset6: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero +; AVX-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; AVX-NEXT: retq +entry: + %B = shufflevector <8 x i16> %A, <8 x i16> zeroinitializer, <8 x i32> + %Z = bitcast <8 x i16> %B to <2 x i64> + ret <2 x i64> %Z +} + +define <4 x i64> @shuf_zext_8i16_to_4i64_offset2(<8 x i16> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: shuf_zext_8i16_to_4i64_offset2: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: shuf_zext_8i16_to_4i64_offset2: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSSE3-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSSE3-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: shuf_zext_8i16_to_4i64_offset2: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; SSE41-NEXT: pmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; SSE41-NEXT: pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; SSE41-NEXT: movdqa %xmm2, %xmm0 +; SSE41-NEXT: retq +; +; AVX1-LABEL: shuf_zext_8i16_to_4i64_offset2: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3] +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuf_zext_8i16_to_4i64_offset2: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,2,2,3] +; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX2-NEXT: retq +entry: + %B = shufflevector <8 x i16> %A, <8 x i16> zeroinitializer, <16 x i32> + %Z = bitcast <16 x i16> %B to <4 x i64> + ret <4 x i64> %Z +} + +define <4 x i32> @shuf_zext_8i16_to_4i32_offset1(<8 x i16> %A) nounwind uwtable readnone ssp { +; SSE-LABEL: shuf_zext_8i16_to_4i32_offset1: +; SSE: # BB#0: # %entry +; SSE-NEXT: psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE-NEXT: retq +; +; AVX-LABEL: shuf_zext_8i16_to_4i32_offset1: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero +; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX-NEXT: retq +entry: + %B = shufflevector <8 x i16> %A, <8 x i16> zeroinitializer, <8 x i32> + %Z = bitcast <8 x i16> %B to <4 x i32> + ret <4 x i32> %Z +} + +define <8 x i32> @shuf_zext_8i16_to_8i32_offset3(<8 x i16> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: shuf_zext_8i16_to_8i32_offset3: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: psrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSE2-NEXT: psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE2-NEXT: retq +; +; SSSE3-LABEL: shuf_zext_8i16_to_8i32_offset3: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: psrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSSE3-NEXT: psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSSE3-NEXT: retq +; +; SSE41-LABEL: shuf_zext_8i16_to_8i32_offset3: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: pslldq {{.*#+}} xmm0 = zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13] +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; SSE41-NEXT: psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE41-NEXT: retq +; +; AVX1-LABEL: shuf_zext_8i16_to_8i32_offset3: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpslldq {{.*#+}} xmm1 = zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13] +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; AVX1-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuf_zext_8i16_to_8i32_offset3: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero +; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX2-NEXT: retq +entry: + %B = shufflevector <8 x i16> %A, <8 x i16> zeroinitializer, <16 x i32> + %Z = bitcast <16 x i16> %B to <8 x i32> + ret <8 x i32> %Z +} + +define <8 x i32> @shuf_zext_16i16_to_8i32_offset8(<16 x i16> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: shuf_zext_16i16_to_8i32_offset8: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE2-NEXT: retq +; +; SSSE3-LABEL: shuf_zext_16i16_to_8i32_offset8: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSSE3-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSSE3-NEXT: retq +; +; SSE41-LABEL: shuf_zext_16i16_to_8i32_offset8: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,2,3,3] +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7] +; SSE41-NEXT: pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero +; SSE41-NEXT: movdqa %xmm2, %xmm1 +; SSE41-NEXT: retq +; +; AVX1-LABEL: shuf_zext_16i16_to_8i32_offset8: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3] +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3],xmm1[4],xmm2[5,6,7] +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuf_zext_16i16_to_8i32_offset8: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 +; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX2-NEXT: retq +entry: + %B = shufflevector <16 x i16> %A, <16 x i16> zeroinitializer, <16 x i32> + %Z = bitcast <16 x i16> %B to <8 x i32> + ret <8 x i32> %Z +} + +define <2 x i64> @shuf_zext_4i32_to_2i64_offset2(<4 x i32> %A) nounwind uwtable readnone ssp { +; SSE-LABEL: shuf_zext_4i32_to_2i64_offset2: +; SSE: # BB#0: # %entry +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSE-NEXT: retq +; +; AVX-LABEL: shuf_zext_4i32_to_2i64_offset2: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX-NEXT: retq +entry: + %B = shufflevector <4 x i32> %A, <4 x i32> zeroinitializer, <4 x i32> + %Z = bitcast <4 x i32> %B to <2 x i64> + ret <2 x i64> %Z +} + +define <4 x i64> @shuf_zext_4i32_to_4i64_offset1(<4 x i32> %A) nounwind uwtable readnone ssp { +; SSE2-LABEL: shuf_zext_4i32_to_4i64_offset1: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [0,0,4294967295,0] +; SSE2-NEXT: pand %xmm1, %xmm0 +; SSE2-NEXT: psrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE2-NEXT: retq +; +; SSSE3-LABEL: shuf_zext_4i32_to_4i64_offset1: +; SSSE3: # BB#0: # %entry +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: movdqa {{.*#+}} xmm0 = [0,0,4294967295,0] +; SSSE3-NEXT: pand %xmm1, %xmm0 +; SSSE3-NEXT: psrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSSE3-NEXT: retq +; +; SSE41-LABEL: shuf_zext_4i32_to_4i64_offset1: +; SSE41: # BB#0: # %entry +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: pxor %xmm0, %xmm0 +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5],xmm0[6,7] +; SSE41-NEXT: psrldq {{.*#+}} xmm1 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE41-NEXT: retq +; +; AVX1-LABEL: shuf_zext_4i32_to_4i64_offset1: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[3],zero,zero,zero +; AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vblendps {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3] +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: shuf_zext_4i32_to_4i64_offset1: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,2,3,3] +; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX2-NEXT: retq +entry: + %B = shufflevector <4 x i32> %A, <4 x i32> zeroinitializer, <8 x i32> + %Z = bitcast <8 x i32> %B to <4 x i64> + ret <4 x i64> %Z +} diff --git a/test/CodeGen/X86/vector-zmov.ll b/test/CodeGen/X86/vector-zmov.ll index 298683559054..e378a3244b4e 100644 --- a/test/CodeGen/X86/vector-zmov.ll +++ b/test/CodeGen/X86/vector-zmov.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=SSE --check-prefix=SSSE3 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41 diff --git a/test/CodeGen/X86/virtual-registers-cleared-in-machine-functions-liveins.ll b/test/CodeGen/X86/virtual-registers-cleared-in-machine-functions-liveins.ll new file mode 100644 index 000000000000..2ff8c3a9028f --- /dev/null +++ b/test/CodeGen/X86/virtual-registers-cleared-in-machine-functions-liveins.ll @@ -0,0 +1,19 @@ +; RUN: llc -mtriple=x86_64-unknown-unknown -o /dev/null -stop-after machine-scheduler %s | FileCheck %s --check-prefix=PRE-RA +; RUN: llc -mtriple=x86_64-unknown-unknown -o /dev/null -stop-after prologepilog %s | FileCheck %s --check-prefix=POST-RA + +; This test verifies that the virtual register references in machine function's +; liveins are cleared after register allocation. + +define i32 @test(i32 %a, i32 %b) { +body: + %c = mul i32 %a, %b + ret i32 %c +} + +; PRE-RA: liveins: +; PRE-RA-NEXT: - { reg: '%edi', virtual-reg: '%0' } +; PRE-RA-NEXT: - { reg: '%esi', virtual-reg: '%1' } + +; POST-RA: liveins: +; POST-RA-NEXT: - { reg: '%edi' } +; POST-RA-NEXT: - { reg: '%esi' } diff --git a/test/CodeGen/X86/vmovq.ll b/test/CodeGen/X86/vmovq.ll new file mode 100644 index 000000000000..45d350c743e2 --- /dev/null +++ b/test/CodeGen/X86/vmovq.ll @@ -0,0 +1,28 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse4.1 | FileCheck %s --check-prefix=SSE +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx | FileCheck %s --check-prefix=AVX + +define <2 x i64> @PR25554(<2 x i64> %v0, <2 x i64> %v1) { +; SSE-LABEL: PR25554: +; SSE: # BB#0: +; SSE-NEXT: movl $1, %eax +; SSE-NEXT: movd %rax, %xmm1 +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6,7] +; SSE-NEXT: paddq %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: PR25554: +; AVX: # BB#0: +; AVX-NEXT: movl $1, %eax +; AVX-NEXT: vmovq %rax, %xmm1 +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6,7] +; AVX-NEXT: vpaddq %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + + %c1 = or <2 x i64> %v0, + %c2 = add <2 x i64> %c1, + ret <2 x i64> %c2 +} + diff --git a/test/CodeGen/X86/vselect-2.ll b/test/CodeGen/X86/vselect-2.ll index fe4cfba08b8a..8e0f4a4ef447 100644 --- a/test/CodeGen/X86/vselect-2.ll +++ b/test/CodeGen/X86/vselect-2.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE41 diff --git a/test/CodeGen/X86/vselect-avx.ll b/test/CodeGen/X86/vselect-avx.ll index de04a097de02..002561042688 100644 --- a/test/CodeGen/X86/vselect-avx.ll +++ b/test/CodeGen/X86/vselect-avx.ll @@ -62,13 +62,15 @@ bb: ; CHECK-LABEL: test3: ; Compute the mask. -; CHECK: vpcmpeqd {{%xmm[0-9]+}}, {{%xmm[0-9]+}}, [[MASK:%xmm[0-9]+]] +; CHECK: vpcmpeqd {{%xmm[0-9]+}}, {{%xmm[0-9]+}}, [[MASK:%xmm[0-9]+]] ; Do not shrink the bit of the mask. -; CHECK-NOT: vpslld $31, [[MASK]], {{%xmm[0-9]+}} +; CHECK-NOT: vpslld $31, [[MASK]], {{%xmm[0-9]+}} ; Use the mask in the blend. -; CHECK-NEXT: vblendvps [[MASK]], %xmm{{[0-9]+}}, %xmm{{[0-9]+}}, %xmm{{[0-9]+}} -; Use the mask in the and. -; CHECK-NEXT: vpand LCPI2_2(%rip), [[MASK]], {{%xmm[0-9]+}} +; CHECK-NEXT: vblendvps [[MASK]], %xmm{{[0-9]+}}, %xmm{{[0-9]+}}, %xmm{{[0-9]+}} +; Shuffle mask to truncate. +; CHECK-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15] +; CHECK: vpshufb %xmm{{[0-9]+}}, %xmm{{[0-9]+}}, %xmm{{[0-9]+}} +; CHECK: vpshufb %xmm{{[0-9]+}}, %xmm{{[0-9]+}}, %xmm{{[0-9]+}} ; CHECK: retq define void @test3(<4 x i32> %induction30, <4 x i16>* %tmp16, <4 x i16>* %tmp17, <4 x i16> %tmp3, <4 x i16> %tmp12) { %tmp6 = srem <4 x i32> %induction30, diff --git a/test/CodeGen/X86/vselect-minmax.ll b/test/CodeGen/X86/vselect-minmax.ll index 5ed687f50576..edf2a442918a 100644 --- a/test/CodeGen/X86/vselect-minmax.ll +++ b/test/CodeGen/X86/vselect-minmax.ll @@ -1,5578 +1,11060 @@ -; RUN: llc -march=x86-64 -mcpu=core2 < %s | FileCheck %s -check-prefix=SSE2 -; RUN: llc -march=x86-64 -mcpu=corei7 < %s | FileCheck %s -check-prefix=SSE4 -; RUN: llc -march=x86-64 -mcpu=corei7-avx < %s | FileCheck %s -check-prefix=AVX1 -; RUN: llc -march=x86-64 -mcpu=core-avx2 -mattr=+avx2 < %s | FileCheck %s -check-prefix=AVX2 -; RUN: llc -march=x86-64 -mcpu=knl < %s | FileCheck %s -check-prefix=AVX2 -check-prefix=AVX512F -; RUN: llc -march=x86-64 -mcpu=skx < %s | FileCheck %s -check-prefix=AVX512BW -check-prefix=AVX512VL -check-prefix=AVX512F - -define void @test1(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <16 x i8>* - %ptr.b = bitcast i8* %gep.b to <16 x i8>* - %load.a = load <16 x i8>, <16 x i8>* %ptr.a, align 2 - %load.b = load <16 x i8>, <16 x i8>* %ptr.b, align 2 - %cmp = icmp slt <16 x i8> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i8> %load.a, <16 x i8> %load.b - store <16 x i8> %sel, <16 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE4 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl | FileCheck %s --check-prefix=AVX --check-prefix=AVX512F +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s --check-prefix=AVX --check-prefix=AVX512F --check-prefix=AVX512BW --check-prefix=AVX512VL + +define <16 x i8> @test1(<16 x i8> %a, <16 x i8> %b) { +; SSE2-LABEL: test1: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtb %xmm0, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test1: -; SSE4: pminsb - -; AVX1-LABEL: test1: -; AVX1: vpminsb - -; AVX2-LABEL: test1: -; AVX2: vpminsb - -; AVX512VL-LABEL: test1: -; AVX512VL: vpminsb -} - -define void @test2(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <16 x i8>* - %ptr.b = bitcast i8* %gep.b to <16 x i8>* - %load.a = load <16 x i8>, <16 x i8>* %ptr.a, align 2 - %load.b = load <16 x i8>, <16 x i8>* %ptr.b, align 2 - %cmp = icmp sle <16 x i8> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i8> %load.a, <16 x i8> %load.b - store <16 x i8> %sel, <16 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsb %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test1: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminsb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp slt <16 x i8> %a, %b + %sel = select <16 x i1> %cmp, <16 x i8> %a, <16 x i8> %b + ret <16 x i8> %sel +} + +define <16 x i8> @test2(<16 x i8> %a, <16 x i8> %b) { +; SSE2-LABEL: test2: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm2 +; SSE2-NEXT: pcmpeqd %xmm3, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test2: -; SSE4: pminsb - -; AVX1-LABEL: test2: -; AVX1: vpminsb - -; AVX2-LABEL: test2: -; AVX2: vpminsb - -; AVX512VL-LABEL: test2: -; AVX512VL: vpminsb -} - -define void @test3(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <16 x i8>* - %ptr.b = bitcast i8* %gep.b to <16 x i8>* - %load.a = load <16 x i8>, <16 x i8>* %ptr.a, align 2 - %load.b = load <16 x i8>, <16 x i8>* %ptr.b, align 2 - %cmp = icmp sgt <16 x i8> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i8> %load.a, <16 x i8> %load.b - store <16 x i8> %sel, <16 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsb %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test2: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminsb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sle <16 x i8> %a, %b + %sel = select <16 x i1> %cmp, <16 x i8> %a, <16 x i8> %b + ret <16 x i8> %sel +} + +define <16 x i8> @test3(<16 x i8> %a, <16 x i8> %b) { +; SSE2-LABEL: test3: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test3: -; SSE4: pmaxsb - -; AVX1-LABEL: test3: -; AVX1: vpmaxsb - -; AVX2-LABEL: test3: -; AVX2: vpmaxsb - -; AVX512VL-LABEL: test3: -; AVX512VL: vpmaxsb -} - -define void @test4(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <16 x i8>* - %ptr.b = bitcast i8* %gep.b to <16 x i8>* - %load.a = load <16 x i8>, <16 x i8>* %ptr.a, align 2 - %load.b = load <16 x i8>, <16 x i8>* %ptr.b, align 2 - %cmp = icmp sge <16 x i8> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i8> %load.a, <16 x i8> %load.b - store <16 x i8> %sel, <16 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsb %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test3: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sgt <16 x i8> %a, %b + %sel = select <16 x i1> %cmp, <16 x i8> %a, <16 x i8> %b + ret <16 x i8> %sel +} + +define <16 x i8> @test4(<16 x i8> %a, <16 x i8> %b) { +; SSE2-LABEL: test4: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pcmpgtb %xmm0, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test4: -; SSE4: pmaxsb - -; AVX1-LABEL: test4: -; AVX1: vpmaxsb - -; AVX2-LABEL: test4: -; AVX2: vpmaxsb - -; AVX512VL-LABEL: test4: -; AVX512VL: vpmaxsb -} - -define void @test5(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <16 x i8>* - %ptr.b = bitcast i8* %gep.b to <16 x i8>* - %load.a = load <16 x i8>, <16 x i8>* %ptr.a, align 2 - %load.b = load <16 x i8>, <16 x i8>* %ptr.b, align 2 - %cmp = icmp ult <16 x i8> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i8> %load.a, <16 x i8> %load.b - store <16 x i8> %sel, <16 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; SSE2-LABEL: test5: -; SSE2: pminub - -; AVX1-LABEL: test5: -; AVX1: vpminub - -; AVX2-LABEL: test5: -; AVX2: vpminub - -; AVX512VL-LABEL: test5: -; AVX512VL: vpminub -} - -define void @test6(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <16 x i8>* - %ptr.b = bitcast i8* %gep.b to <16 x i8>* - %load.a = load <16 x i8>, <16 x i8>* %ptr.a, align 2 - %load.b = load <16 x i8>, <16 x i8>* %ptr.b, align 2 - %cmp = icmp ule <16 x i8> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i8> %load.a, <16 x i8> %load.b - store <16 x i8> %sel, <16 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; SSE2-LABEL: test6: -; SSE2: pminub - -; AVX1-LABEL: test6: -; AVX1: vpminub - -; AVX2-LABEL: test6: -; AVX2: vpminub - -; AVX512VL-LABEL: test6: -; AVX512VL: vpminub -} - -define void @test7(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <16 x i8>* - %ptr.b = bitcast i8* %gep.b to <16 x i8>* - %load.a = load <16 x i8>, <16 x i8>* %ptr.a, align 2 - %load.b = load <16 x i8>, <16 x i8>* %ptr.b, align 2 - %cmp = icmp ugt <16 x i8> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i8> %load.a, <16 x i8> %load.b - store <16 x i8> %sel, <16 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; SSE2-LABEL: test7: -; SSE2: pmaxub - -; AVX1-LABEL: test7: -; AVX1: vpmaxub - -; AVX2-LABEL: test7: -; AVX2: vpmaxub - -; AVX512VL-LABEL: test7: -; AVX512VL: vpmaxub -} - -define void @test8(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <16 x i8>* - %ptr.b = bitcast i8* %gep.b to <16 x i8>* - %load.a = load <16 x i8>, <16 x i8>* %ptr.a, align 2 - %load.b = load <16 x i8>, <16 x i8>* %ptr.b, align 2 - %cmp = icmp uge <16 x i8> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i8> %load.a, <16 x i8> %load.b - store <16 x i8> %sel, <16 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; SSE2-LABEL: test8: -; SSE2: pmaxub - -; AVX1-LABEL: test8: -; AVX1: vpmaxub - -; AVX2-LABEL: test8: -; AVX2: vpmaxub - -; AVX512VL-LABEL: test8: -; AVX512VL: vpmaxub -} - -define void @test9(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <8 x i16>* - %ptr.b = bitcast i16* %gep.b to <8 x i16>* - %load.a = load <8 x i16>, <8 x i16>* %ptr.a, align 2 - %load.b = load <8 x i16>, <8 x i16>* %ptr.b, align 2 - %cmp = icmp slt <8 x i16> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i16> %load.a, <8 x i16> %load.b - store <8 x i16> %sel, <8 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; SSE2-LABEL: test9: -; SSE2: pminsw - -; AVX1-LABEL: test9: -; AVX1: vpminsw - -; AVX2-LABEL: test9: -; AVX2: vpminsw - -; AVX512VL-LABEL: test9: -; AVX512VL: vpminsw -} - -define void @test10(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <8 x i16>* - %ptr.b = bitcast i16* %gep.b to <8 x i16>* - %load.a = load <8 x i16>, <8 x i16>* %ptr.a, align 2 - %load.b = load <8 x i16>, <8 x i16>* %ptr.b, align 2 - %cmp = icmp sle <8 x i16> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i16> %load.a, <8 x i16> %load.b - store <8 x i16> %sel, <8 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; SSE2-LABEL: test10: -; SSE2: pminsw - -; AVX1-LABEL: test10: -; AVX1: vpminsw - -; AVX2-LABEL: test10: -; AVX2: vpminsw - -; AVX512VL-LABEL: test10: -; AVX512VL: vpminsw -} - -define void @test11(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <8 x i16>* - %ptr.b = bitcast i16* %gep.b to <8 x i16>* - %load.a = load <8 x i16>, <8 x i16>* %ptr.a, align 2 - %load.b = load <8 x i16>, <8 x i16>* %ptr.b, align 2 - %cmp = icmp sgt <8 x i16> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i16> %load.a, <8 x i16> %load.b - store <8 x i16> %sel, <8 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; SSE2-LABEL: test11: -; SSE2: pmaxsw - -; AVX1-LABEL: test11: -; AVX1: vpmaxsw - -; AVX2-LABEL: test11: -; AVX2: vpmaxsw - -; AVX512VL-LABEL: test11: -; AVX512VL: vpmaxsw -} - -define void @test12(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <8 x i16>* - %ptr.b = bitcast i16* %gep.b to <8 x i16>* - %load.a = load <8 x i16>, <8 x i16>* %ptr.a, align 2 - %load.b = load <8 x i16>, <8 x i16>* %ptr.b, align 2 - %cmp = icmp sge <8 x i16> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i16> %load.a, <8 x i16> %load.b - store <8 x i16> %sel, <8 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; SSE2-LABEL: test12: -; SSE2: pmaxsw - -; AVX1-LABEL: test12: -; AVX1: vpmaxsw - -; AVX2-LABEL: test12: -; AVX2: vpmaxsw - -; AVX512VL-LABEL: test12: -; AVX512VL: vpmaxsw -} - -define void @test13(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <8 x i16>* - %ptr.b = bitcast i16* %gep.b to <8 x i16>* - %load.a = load <8 x i16>, <8 x i16>* %ptr.a, align 2 - %load.b = load <8 x i16>, <8 x i16>* %ptr.b, align 2 - %cmp = icmp ult <8 x i16> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i16> %load.a, <8 x i16> %load.b - store <8 x i16> %sel, <8 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsb %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test4: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sge <16 x i8> %a, %b + %sel = select <16 x i1> %cmp, <16 x i8> %a, <16 x i8> %b + ret <16 x i8> %sel +} + +define <16 x i8> @test5(<16 x i8> %a, <16 x i8> %b) { +; SSE-LABEL: test5: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminub %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test5: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminub %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ult <16 x i8> %a, %b + %sel = select <16 x i1> %cmp, <16 x i8> %a, <16 x i8> %b + ret <16 x i8> %sel +} + +define <16 x i8> @test6(<16 x i8> %a, <16 x i8> %b) { +; SSE-LABEL: test6: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminub %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test6: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminub %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ule <16 x i8> %a, %b + %sel = select <16 x i1> %cmp, <16 x i8> %a, <16 x i8> %b + ret <16 x i8> %sel +} + +define <16 x i8> @test7(<16 x i8> %a, <16 x i8> %b) { +; SSE-LABEL: test7: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxub %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test7: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxub %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ugt <16 x i8> %a, %b + %sel = select <16 x i1> %cmp, <16 x i8> %a, <16 x i8> %b + ret <16 x i8> %sel +} + +define <16 x i8> @test8(<16 x i8> %a, <16 x i8> %b) { +; SSE-LABEL: test8: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxub %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test8: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxub %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp uge <16 x i8> %a, %b + %sel = select <16 x i1> %cmp, <16 x i8> %a, <16 x i8> %b + ret <16 x i8> %sel +} + +define <8 x i16> @test9(<8 x i16> %a, <8 x i16> %b) { +; SSE-LABEL: test9: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminsw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test9: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminsw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp slt <8 x i16> %a, %b + %sel = select <8 x i1> %cmp, <8 x i16> %a, <8 x i16> %b + ret <8 x i16> %sel +} + +define <8 x i16> @test10(<8 x i16> %a, <8 x i16> %b) { +; SSE-LABEL: test10: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminsw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test10: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminsw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sle <8 x i16> %a, %b + %sel = select <8 x i1> %cmp, <8 x i16> %a, <8 x i16> %b + ret <8 x i16> %sel +} + +define <8 x i16> @test11(<8 x i16> %a, <8 x i16> %b) { +; SSE-LABEL: test11: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxsw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test11: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sgt <8 x i16> %a, %b + %sel = select <8 x i1> %cmp, <8 x i16> %a, <8 x i16> %b + ret <8 x i16> %sel +} + +define <8 x i16> @test12(<8 x i16> %a, <8 x i16> %b) { +; SSE-LABEL: test12: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxsw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test12: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sge <8 x i16> %a, %b + %sel = select <8 x i1> %cmp, <8 x i16> %a, <8 x i16> %b + ret <8 x i16> %sel +} + +define <8 x i16> @test13(<8 x i16> %a, <8 x i16> %b) { +; SSE2-LABEL: test13: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtw %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test13: -; SSE4: pminuw - -; AVX1-LABEL: test13: -; AVX1: vpminuw - -; AVX2-LABEL: test13: -; AVX2: vpminuw - -; AVX512VL-LABEL: test13: -; AVX512VL: vpminuw -} - -define void @test14(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <8 x i16>* - %ptr.b = bitcast i16* %gep.b to <8 x i16>* - %load.a = load <8 x i16>, <8 x i16>* %ptr.a, align 2 - %load.b = load <8 x i16>, <8 x i16>* %ptr.b, align 2 - %cmp = icmp ule <8 x i16> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i16> %load.a, <8 x i16> %load.b - store <8 x i16> %sel, <8 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminuw %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test13: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminuw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ult <8 x i16> %a, %b + %sel = select <8 x i1> %cmp, <8 x i16> %a, <8 x i16> %b + ret <8 x i16> %sel +} + +define <8 x i16> @test14(<8 x i16> %a, <8 x i16> %b) { +; SSE2-LABEL: test14: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: psubusw %xmm1, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm3 +; SSE2-NEXT: pcmpeqw %xmm2, %xmm3 +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test14: -; SSE4: pminuw - -; AVX1-LABEL: test14: -; AVX1: vpminuw - -; AVX2-LABEL: test14: -; AVX2: vpminuw - -; AVX512VL-LABEL: test14: -; AVX512VL: vpminuw -} - -define void @test15(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <8 x i16>* - %ptr.b = bitcast i16* %gep.b to <8 x i16>* - %load.a = load <8 x i16>, <8 x i16>* %ptr.a, align 2 - %load.b = load <8 x i16>, <8 x i16>* %ptr.b, align 2 - %cmp = icmp ugt <8 x i16> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i16> %load.a, <8 x i16> %load.b - store <8 x i16> %sel, <8 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminuw %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test14: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminuw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ule <8 x i16> %a, %b + %sel = select <8 x i1> %cmp, <8 x i16> %a, <8 x i16> %b + ret <8 x i16> %sel +} + +define <8 x i16> @test15(<8 x i16> %a, <8 x i16> %b) { +; SSE2-LABEL: test15: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtw %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test15: -; SSE4: pmaxuw - -; AVX1-LABEL: test15: -; AVX1: vpmaxuw - -; AVX2-LABEL: test15: -; AVX2: vpmaxuw - -; AVX512VL-LABEL: test15: -; AVX512VL: vpmaxuw -} - -define void @test16(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <8 x i16>* - %ptr.b = bitcast i16* %gep.b to <8 x i16>* - %load.a = load <8 x i16>, <8 x i16>* %ptr.a, align 2 - %load.b = load <8 x i16>, <8 x i16>* %ptr.b, align 2 - %cmp = icmp uge <8 x i16> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i16> %load.a, <8 x i16> %load.b - store <8 x i16> %sel, <8 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxuw %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test15: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxuw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ugt <8 x i16> %a, %b + %sel = select <8 x i1> %cmp, <8 x i16> %a, <8 x i16> %b + ret <8 x i16> %sel +} + +define <8 x i16> @test16(<8 x i16> %a, <8 x i16> %b) { +; SSE2-LABEL: test16: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: psubusw %xmm0, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm3 +; SSE2-NEXT: pcmpeqw %xmm2, %xmm3 +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test16: -; SSE4: pmaxuw - -; AVX1-LABEL: test16: -; AVX1: vpmaxuw - -; AVX2-LABEL: test16: -; AVX2: vpmaxuw - -; AVX512VL-LABEL: test16: -; AVX512VL: vpmaxuw -} - -define void @test17(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i32>* - %ptr.b = bitcast i32* %gep.b to <4 x i32>* - %load.a = load <4 x i32>, <4 x i32>* %ptr.a, align 2 - %load.b = load <4 x i32>, <4 x i32>* %ptr.b, align 2 - %cmp = icmp slt <4 x i32> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i32> %load.a, <4 x i32> %load.b - store <4 x i32> %sel, <4 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 4 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxuw %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test16: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxuw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp uge <8 x i16> %a, %b + %sel = select <8 x i1> %cmp, <8 x i16> %a, <8 x i16> %b + ret <8 x i16> %sel +} + +define <4 x i32> @test17(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: test17: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test17: -; SSE4: pminsd - -; AVX1-LABEL: test17: -; AVX1: vpminsd - -; AVX2-LABEL: test17: -; AVX2: vpminsd - -; AVX512VL-LABEL: test17: -; AVX512VL: vpminsd -} - -define void @test18(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i32>* - %ptr.b = bitcast i32* %gep.b to <4 x i32>* - %load.a = load <4 x i32>, <4 x i32>* %ptr.a, align 2 - %load.b = load <4 x i32>, <4 x i32>* %ptr.b, align 2 - %cmp = icmp sle <4 x i32> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i32> %load.a, <4 x i32> %load.b - store <4 x i32> %sel, <4 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 4 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsd %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test17: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminsd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp slt <4 x i32> %a, %b + %sel = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %b + ret <4 x i32> %sel +} + +define <4 x i32> @test18(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: test18: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE2-NEXT: pcmpeqd %xmm3, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test18: -; SSE4: pminsd - -; AVX1-LABEL: test18: -; AVX1: vpminsd - -; AVX2-LABEL: test18: -; AVX2: vpminsd - -; AVX512VL-LABEL: test18: -; AVX512VL: vpminsd -} - -define void @test19(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i32>* - %ptr.b = bitcast i32* %gep.b to <4 x i32>* - %load.a = load <4 x i32>, <4 x i32>* %ptr.a, align 2 - %load.b = load <4 x i32>, <4 x i32>* %ptr.b, align 2 - %cmp = icmp sgt <4 x i32> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i32> %load.a, <4 x i32> %load.b - store <4 x i32> %sel, <4 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 4 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsd %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test18: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminsd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sle <4 x i32> %a, %b + %sel = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %b + ret <4 x i32> %sel +} + +define <4 x i32> @test19(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: test19: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test19: -; SSE4: pmaxsd - -; AVX1-LABEL: test19: -; AVX1: vpmaxsd - -; AVX2-LABEL: test19: -; AVX2: vpmaxsd - -; AVX512VL-LABEL: test19: -; AVX512VL: vpmaxsd -} - -define void @test20(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i32>* - %ptr.b = bitcast i32* %gep.b to <4 x i32>* - %load.a = load <4 x i32>, <4 x i32>* %ptr.a, align 2 - %load.b = load <4 x i32>, <4 x i32>* %ptr.b, align 2 - %cmp = icmp sge <4 x i32> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i32> %load.a, <4 x i32> %load.b - store <4 x i32> %sel, <4 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 4 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsd %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test19: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sgt <4 x i32> %a, %b + %sel = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %b + ret <4 x i32> %sel +} + +define <4 x i32> @test20(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: test20: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test20: -; SSE4: pmaxsd - -; AVX1-LABEL: test20: -; AVX1: vpmaxsd - -; AVX2-LABEL: test20: -; AVX2: vpmaxsd - -; AVX512VL-LABEL: test20: -; AVX512VL: vpmaxsd -} - -define void @test21(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i32>* - %ptr.b = bitcast i32* %gep.b to <4 x i32>* - %load.a = load <4 x i32>, <4 x i32>* %ptr.a, align 2 - %load.b = load <4 x i32>, <4 x i32>* %ptr.b, align 2 - %cmp = icmp ult <4 x i32> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i32> %load.a, <4 x i32> %load.b - store <4 x i32> %sel, <4 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 4 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsd %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test20: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sge <4 x i32> %a, %b + %sel = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %b + ret <4 x i32> %sel +} + +define <4 x i32> @test21(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: test21: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test21: -; SSE4: pminud - -; AVX1-LABEL: test21: -; AVX1: vpminud - -; AVX2-LABEL: test21: -; AVX2: vpminud - -; AVX512VL-LABEL: test21: -; AVX512VL: vpminud -} - -define void @test22(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i32>* - %ptr.b = bitcast i32* %gep.b to <4 x i32>* - %load.a = load <4 x i32>, <4 x i32>* %ptr.a, align 2 - %load.b = load <4 x i32>, <4 x i32>* %ptr.b, align 2 - %cmp = icmp ule <4 x i32> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i32> %load.a, <4 x i32> %load.b - store <4 x i32> %sel, <4 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 4 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminud %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test21: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminud %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ult <4 x i32> %a, %b + %sel = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %b + ret <4 x i32> %sel +} + +define <4 x i32> @test22(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: test22: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pxor %xmm0, %xmm3 +; SSE2-NEXT: pcmpgtd %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test22: -; SSE4: pminud - -; AVX1-LABEL: test22: -; AVX1: vpminud - -; AVX2-LABEL: test22: -; AVX2: vpminud - -; AVX512VL-LABEL: test22: -; AVX512VL: vpminud -} - -define void @test23(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i32>* - %ptr.b = bitcast i32* %gep.b to <4 x i32>* - %load.a = load <4 x i32>, <4 x i32>* %ptr.a, align 2 - %load.b = load <4 x i32>, <4 x i32>* %ptr.b, align 2 - %cmp = icmp ugt <4 x i32> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i32> %load.a, <4 x i32> %load.b - store <4 x i32> %sel, <4 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 4 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminud %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test22: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminud %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ule <4 x i32> %a, %b + %sel = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %b + ret <4 x i32> %sel +} + +define <4 x i32> @test23(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: test23: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test23: -; SSE4: pmaxud - -; AVX1-LABEL: test23: -; AVX1: vpmaxud - -; AVX2-LABEL: test23: -; AVX2: vpmaxud - -; AVX512VL-LABEL: test23: -; AVX512VL: vpmaxud -} - -define void @test24(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i32>* - %ptr.b = bitcast i32* %gep.b to <4 x i32>* - %load.a = load <4 x i32>, <4 x i32>* %ptr.a, align 2 - %load.b = load <4 x i32>, <4 x i32>* %ptr.b, align 2 - %cmp = icmp uge <4 x i32> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i32> %load.a, <4 x i32> %load.b - store <4 x i32> %sel, <4 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 4 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxud %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test23: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxud %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ugt <4 x i32> %a, %b + %sel = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %b + ret <4 x i32> %sel +} + +define <4 x i32> @test24(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: test24: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pxor %xmm1, %xmm3 +; SSE2-NEXT: pcmpgtd %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test24: -; SSE4: pmaxud - -; AVX1-LABEL: test24: -; AVX1: vpmaxud - -; AVX2-LABEL: test24: -; AVX2: vpmaxud - -; AVX512VL-LABEL: test24: -; AVX512VL: vpmaxud -} - -define void @test25(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <32 x i8>* - %ptr.b = bitcast i8* %gep.b to <32 x i8>* - %load.a = load <32 x i8>, <32 x i8>* %ptr.a, align 2 - %load.b = load <32 x i8>, <32 x i8>* %ptr.b, align 2 - %cmp = icmp slt <32 x i8> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i8> %load.a, <32 x i8> %load.b - store <32 x i8> %sel, <32 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxud %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test24: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxud %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp uge <4 x i32> %a, %b + %sel = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %b + ret <4 x i32> %sel +} + +define <32 x i8> @test25(<32 x i8> %a, <32 x i8> %b) { +; SSE2-LABEL: test25: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm4 +; SSE2-NEXT: movdqa %xmm2, %xmm5 +; SSE2-NEXT: pcmpgtb %xmm0, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test25: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsb %xmm2, %xmm0 +; SSE4-NEXT: pminsb %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test25: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsb %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test25: -; AVX2: vpminsb - -; AVX512VL-LABEL: test25: -; AVX512VL: vpminsb -} - -define void @test26(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <32 x i8>* - %ptr.b = bitcast i8* %gep.b to <32 x i8>* - %load.a = load <32 x i8>, <32 x i8>* %ptr.a, align 2 - %load.b = load <32 x i8>, <32 x i8>* %ptr.b, align 2 - %cmp = icmp sle <32 x i8> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i8> %load.a, <32 x i8> %load.b - store <32 x i8> %sel, <32 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsb %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test25: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsb %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp slt <32 x i8> %a, %b + %sel = select <32 x i1> %cmp, <32 x i8> %a, <32 x i8> %b + ret <32 x i8> %sel +} + +define <32 x i8> @test26(<32 x i8> %a, <32 x i8> %b) { +; SSE2-LABEL: test26: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm6 +; SSE2-NEXT: pcmpgtb %xmm3, %xmm6 +; SSE2-NEXT: pcmpeqd %xmm7, %xmm7 +; SSE2-NEXT: movdqa %xmm6, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pcmpgtb %xmm2, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm7 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: pandn %xmm2, %xmm7 +; SSE2-NEXT: por %xmm7, %xmm5 +; SSE2-NEXT: pandn %xmm1, %xmm6 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test26: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsb %xmm2, %xmm0 +; SSE4-NEXT: pminsb %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test26: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsb %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test26: -; AVX2: vpminsb - -; AVX512VL-LABEL: test26: -; AVX512VL: vpminsb -} - -define void @test27(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <32 x i8>* - %ptr.b = bitcast i8* %gep.b to <32 x i8>* - %load.a = load <32 x i8>, <32 x i8>* %ptr.a, align 2 - %load.b = load <32 x i8>, <32 x i8>* %ptr.b, align 2 - %cmp = icmp sgt <32 x i8> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i8> %load.a, <32 x i8> %load.b - store <32 x i8> %sel, <32 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsb %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test26: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsb %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sle <32 x i8> %a, %b + %sel = select <32 x i1> %cmp, <32 x i8> %a, <32 x i8> %b + ret <32 x i8> %sel +} + +define <32 x i8> @test27(<32 x i8> %a, <32 x i8> %b) { +; SSE2-LABEL: test27: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pcmpgtb %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pcmpgtb %xmm2, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm0, %xmm5 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm1, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test27: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsb %xmm2, %xmm0 +; SSE4-NEXT: pmaxsb %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test27: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsb %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test27: -; AVX2: vpmaxsb - -; AVX512VL-LABEL: test27: -; AVX512VL: vpmaxsb -} - -define void @test28(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <32 x i8>* - %ptr.b = bitcast i8* %gep.b to <32 x i8>* - %load.a = load <32 x i8>, <32 x i8>* %ptr.a, align 2 - %load.b = load <32 x i8>, <32 x i8>* %ptr.b, align 2 - %cmp = icmp sge <32 x i8> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i8> %load.a, <32 x i8> %load.b - store <32 x i8> %sel, <32 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test27: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sgt <32 x i8> %a, %b + %sel = select <32 x i1> %cmp, <32 x i8> %a, <32 x i8> %b + ret <32 x i8> %sel +} + +define <32 x i8> @test28(<32 x i8> %a, <32 x i8> %b) { +; SSE2-LABEL: test28: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm6 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm2, %xmm7 +; SSE2-NEXT: pcmpgtb %xmm0, %xmm7 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm7 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm7, %xmm4 +; SSE2-NEXT: pandn %xmm1, %xmm6 +; SSE2-NEXT: pandn %xmm3, %xmm5 +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test28: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsb %xmm2, %xmm0 +; SSE4-NEXT: pmaxsb %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test28: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsb %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test28: -; AVX2: vpmaxsb - -; AVX512VL-LABEL: test28: -; AVX512VL: vpmaxsb -} - -define void @test29(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <32 x i8>* - %ptr.b = bitcast i8* %gep.b to <32 x i8>* - %load.a = load <32 x i8>, <32 x i8>* %ptr.a, align 2 - %load.b = load <32 x i8>, <32 x i8>* %ptr.b, align 2 - %cmp = icmp ult <32 x i8> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i8> %load.a, <32 x i8> %load.b - store <32 x i8> %sel, <32 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test28: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sge <32 x i8> %a, %b + %sel = select <32 x i1> %cmp, <32 x i8> %a, <32 x i8> %b + ret <32 x i8> %sel +} + +define <32 x i8> @test29(<32 x i8> %a, <32 x i8> %b) { +; SSE-LABEL: test29: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminub %xmm2, %xmm0 +; SSE-NEXT: pminub %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: test29: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminub %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminub %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test29: -; AVX2: vpminub - -; AVX512VL-LABEL: test29: -; AVX512VL: vpminub -} - -define void @test30(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <32 x i8>* - %ptr.b = bitcast i8* %gep.b to <32 x i8>* - %load.a = load <32 x i8>, <32 x i8>* %ptr.a, align 2 - %load.b = load <32 x i8>, <32 x i8>* %ptr.b, align 2 - %cmp = icmp ule <32 x i8> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i8> %load.a, <32 x i8> %load.b - store <32 x i8> %sel, <32 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminub %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test29: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminub %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ult <32 x i8> %a, %b + %sel = select <32 x i1> %cmp, <32 x i8> %a, <32 x i8> %b + ret <32 x i8> %sel +} + +define <32 x i8> @test30(<32 x i8> %a, <32 x i8> %b) { +; SSE-LABEL: test30: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminub %xmm2, %xmm0 +; SSE-NEXT: pminub %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: test30: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminub %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminub %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test30: -; AVX2: vpminub - -; AVX512VL-LABEL: test30: -; AVX512VL: vpminub -} - -define void @test31(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <32 x i8>* - %ptr.b = bitcast i8* %gep.b to <32 x i8>* - %load.a = load <32 x i8>, <32 x i8>* %ptr.a, align 2 - %load.b = load <32 x i8>, <32 x i8>* %ptr.b, align 2 - %cmp = icmp ugt <32 x i8> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i8> %load.a, <32 x i8> %load.b - store <32 x i8> %sel, <32 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminub %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test30: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminub %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ule <32 x i8> %a, %b + %sel = select <32 x i1> %cmp, <32 x i8> %a, <32 x i8> %b + ret <32 x i8> %sel +} + +define <32 x i8> @test31(<32 x i8> %a, <32 x i8> %b) { +; SSE-LABEL: test31: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxub %xmm2, %xmm0 +; SSE-NEXT: pmaxub %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: test31: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxub %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxub %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test31: -; AVX2: vpmaxub - -; AVX512VL-LABEL: test31: -; AVX512VL: vpmaxub -} - -define void @test32(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <32 x i8>* - %ptr.b = bitcast i8* %gep.b to <32 x i8>* - %load.a = load <32 x i8>, <32 x i8>* %ptr.a, align 2 - %load.b = load <32 x i8>, <32 x i8>* %ptr.b, align 2 - %cmp = icmp uge <32 x i8> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i8> %load.a, <32 x i8> %load.b - store <32 x i8> %sel, <32 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxub %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test31: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxub %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ugt <32 x i8> %a, %b + %sel = select <32 x i1> %cmp, <32 x i8> %a, <32 x i8> %b + ret <32 x i8> %sel +} + +define <32 x i8> @test32(<32 x i8> %a, <32 x i8> %b) { +; SSE-LABEL: test32: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxub %xmm2, %xmm0 +; SSE-NEXT: pmaxub %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: test32: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxub %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxub %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test32: -; AVX2: vpmaxub - -; AVX512VL-LABEL: test32: -; AVX512VL: vpmaxub -} - -define void @test33(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <16 x i16>* - %ptr.b = bitcast i16* %gep.b to <16 x i16>* - %load.a = load <16 x i16>, <16 x i16>* %ptr.a, align 2 - %load.b = load <16 x i16>, <16 x i16>* %ptr.b, align 2 - %cmp = icmp slt <16 x i16> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i16> %load.a, <16 x i16> %load.b - store <16 x i16> %sel, <16 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxub %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test32: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxub %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp uge <32 x i8> %a, %b + %sel = select <32 x i1> %cmp, <32 x i8> %a, <32 x i8> %b + ret <32 x i8> %sel +} + +define <16 x i16> @test33(<16 x i16> %a, <16 x i16> %b) { +; SSE-LABEL: test33: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminsw %xmm2, %xmm0 +; SSE-NEXT: pminsw %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: test33: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test33: -; AVX2: vpminsw - -; AVX512VL-LABEL: test33: -; AVX512VL: vpminsw -} - -define void @test34(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <16 x i16>* - %ptr.b = bitcast i16* %gep.b to <16 x i16>* - %load.a = load <16 x i16>, <16 x i16>* %ptr.a, align 2 - %load.b = load <16 x i16>, <16 x i16>* %ptr.b, align 2 - %cmp = icmp sle <16 x i16> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i16> %load.a, <16 x i16> %load.b - store <16 x i16> %sel, <16 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test33: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsw %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp slt <16 x i16> %a, %b + %sel = select <16 x i1> %cmp, <16 x i16> %a, <16 x i16> %b + ret <16 x i16> %sel +} + +define <16 x i16> @test34(<16 x i16> %a, <16 x i16> %b) { +; SSE-LABEL: test34: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminsw %xmm2, %xmm0 +; SSE-NEXT: pminsw %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: test34: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test34: -; AVX2: vpminsw - -; AVX512VL-LABEL: test34: -; AVX512VL: vpminsw -} - -define void @test35(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <16 x i16>* - %ptr.b = bitcast i16* %gep.b to <16 x i16>* - %load.a = load <16 x i16>, <16 x i16>* %ptr.a, align 2 - %load.b = load <16 x i16>, <16 x i16>* %ptr.b, align 2 - %cmp = icmp sgt <16 x i16> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i16> %load.a, <16 x i16> %load.b - store <16 x i16> %sel, <16 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test34: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsw %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sle <16 x i16> %a, %b + %sel = select <16 x i1> %cmp, <16 x i16> %a, <16 x i16> %b + ret <16 x i16> %sel +} + +define <16 x i16> @test35(<16 x i16> %a, <16 x i16> %b) { +; SSE-LABEL: test35: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxsw %xmm2, %xmm0 +; SSE-NEXT: pmaxsw %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: test35: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test35: -; AVX2: vpmaxsw - -; AVX512VL-LABEL: test35: -; AVX512VL: vpmaxsw -} - -define void @test36(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <16 x i16>* - %ptr.b = bitcast i16* %gep.b to <16 x i16>* - %load.a = load <16 x i16>, <16 x i16>* %ptr.a, align 2 - %load.b = load <16 x i16>, <16 x i16>* %ptr.b, align 2 - %cmp = icmp sge <16 x i16> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i16> %load.a, <16 x i16> %load.b - store <16 x i16> %sel, <16 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test35: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sgt <16 x i16> %a, %b + %sel = select <16 x i1> %cmp, <16 x i16> %a, <16 x i16> %b + ret <16 x i16> %sel +} + +define <16 x i16> @test36(<16 x i16> %a, <16 x i16> %b) { +; SSE-LABEL: test36: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxsw %xmm2, %xmm0 +; SSE-NEXT: pmaxsw %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: test36: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test36: -; AVX2: vpmaxsw - -; AVX512VL-LABEL: test36: -; AVX512VL: vpmaxsw -} - -define void @test37(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <16 x i16>* - %ptr.b = bitcast i16* %gep.b to <16 x i16>* - %load.a = load <16 x i16>, <16 x i16>* %ptr.a, align 2 - %load.b = load <16 x i16>, <16 x i16>* %ptr.b, align 2 - %cmp = icmp ult <16 x i16> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i16> %load.a, <16 x i16> %load.b - store <16 x i16> %sel, <16 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test36: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sge <16 x i16> %a, %b + %sel = select <16 x i1> %cmp, <16 x i16> %a, <16 x i16> %b + ret <16 x i16> %sel +} + +define <16 x i16> @test37(<16 x i16> %a, <16 x i16> %b) { +; SSE2-LABEL: test37: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: pcmpgtw %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: pxor %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtw %xmm5, %xmm4 +; SSE2-NEXT: pand %xmm4, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm4, %xmm0 +; SSE2-NEXT: pand %xmm6, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm6 +; SSE2-NEXT: por %xmm6, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test37: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminuw %xmm2, %xmm0 +; SSE4-NEXT: pminuw %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test37: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminuw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminuw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test37: -; AVX2: vpminuw - -; AVX512VL-LABEL: test37: -; AVX512VL: vpminuw -} - -define void @test38(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <16 x i16>* - %ptr.b = bitcast i16* %gep.b to <16 x i16>* - %load.a = load <16 x i16>, <16 x i16>* %ptr.a, align 2 - %load.b = load <16 x i16>, <16 x i16>* %ptr.b, align 2 - %cmp = icmp ule <16 x i16> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i16> %load.a, <16 x i16> %load.b - store <16 x i16> %sel, <16 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminuw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test37: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminuw %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ult <16 x i16> %a, %b + %sel = select <16 x i1> %cmp, <16 x i16> %a, <16 x i16> %b + ret <16 x i16> %sel +} + +define <16 x i16> @test38(<16 x i16> %a, <16 x i16> %b) { +; SSE2-LABEL: test38: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: psubusw %xmm3, %xmm4 +; SSE2-NEXT: pxor %xmm6, %xmm6 +; SSE2-NEXT: pcmpeqw %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: psubusw %xmm2, %xmm5 +; SSE2-NEXT: pcmpeqw %xmm6, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm0, %xmm5 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm1, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test38: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminuw %xmm2, %xmm0 +; SSE4-NEXT: pminuw %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test38: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminuw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminuw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test38: -; AVX2: vpminuw - -; AVX512VL-LABEL: test38: -; AVX512VL: vpminuw -} - -define void @test39(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <16 x i16>* - %ptr.b = bitcast i16* %gep.b to <16 x i16>* - %load.a = load <16 x i16>, <16 x i16>* %ptr.a, align 2 - %load.b = load <16 x i16>, <16 x i16>* %ptr.b, align 2 - %cmp = icmp ugt <16 x i16> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i16> %load.a, <16 x i16> %load.b - store <16 x i16> %sel, <16 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminuw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test38: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminuw %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ule <16 x i16> %a, %b + %sel = select <16 x i1> %cmp, <16 x i16> %a, <16 x i16> %b + ret <16 x i16> %sel +} + +define <16 x i16> @test39(<16 x i16> %a, <16 x i16> %b) { +; SSE2-LABEL: test39: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pxor %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: pcmpgtw %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm2, %xmm6 +; SSE2-NEXT: pxor %xmm5, %xmm6 +; SSE2-NEXT: pxor %xmm0, %xmm5 +; SSE2-NEXT: pcmpgtw %xmm6, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm1, %xmm4 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test39: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxuw %xmm2, %xmm0 +; SSE4-NEXT: pmaxuw %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test39: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxuw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxuw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test39: -; AVX2: vpmaxuw - -; AVX512VL-LABEL: test39: -; AVX512VL: vpmaxuw -} - -define void @test40(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <16 x i16>* - %ptr.b = bitcast i16* %gep.b to <16 x i16>* - %load.a = load <16 x i16>, <16 x i16>* %ptr.a, align 2 - %load.b = load <16 x i16>, <16 x i16>* %ptr.b, align 2 - %cmp = icmp uge <16 x i16> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i16> %load.a, <16 x i16> %load.b - store <16 x i16> %sel, <16 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxuw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test39: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxuw %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ugt <16 x i16> %a, %b + %sel = select <16 x i1> %cmp, <16 x i16> %a, <16 x i16> %b + ret <16 x i16> %sel +} + +define <16 x i16> @test40(<16 x i16> %a, <16 x i16> %b) { +; SSE2-LABEL: test40: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: psubusw %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm5, %xmm5 +; SSE2-NEXT: pcmpeqw %xmm5, %xmm4 +; SSE2-NEXT: movdqa %xmm2, %xmm6 +; SSE2-NEXT: psubusw %xmm0, %xmm6 +; SSE2-NEXT: pcmpeqw %xmm5, %xmm6 +; SSE2-NEXT: pand %xmm6, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm6 +; SSE2-NEXT: por %xmm6, %xmm0 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test40: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxuw %xmm2, %xmm0 +; SSE4-NEXT: pmaxuw %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test40: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxuw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxuw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test40: -; AVX2: vpmaxuw - -; AVX512VL-LABEL: test40: -; AVX512VL: vpmaxuw -} - -define void @test41(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i32>* - %ptr.b = bitcast i32* %gep.b to <8 x i32>* - %load.a = load <8 x i32>, <8 x i32>* %ptr.a, align 2 - %load.b = load <8 x i32>, <8 x i32>* %ptr.b, align 2 - %cmp = icmp slt <8 x i32> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i32> %load.a, <8 x i32> %load.b - store <8 x i32> %sel, <8 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxuw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test40: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxuw %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp uge <16 x i16> %a, %b + %sel = select <16 x i1> %cmp, <16 x i16> %a, <16 x i16> %b + ret <16 x i16> %sel +} + +define <8 x i32> @test41(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: test41: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm4 +; SSE2-NEXT: movdqa %xmm2, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test41: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsd %xmm2, %xmm0 +; SSE4-NEXT: pminsd %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test41: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsd %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test41: -; AVX2: vpminsd - -; AVX512VL-LABEL: test41: -; AVX512VL: vpminsd -} - -define void @test42(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i32>* - %ptr.b = bitcast i32* %gep.b to <8 x i32>* - %load.a = load <8 x i32>, <8 x i32>* %ptr.a, align 2 - %load.b = load <8 x i32>, <8 x i32>* %ptr.b, align 2 - %cmp = icmp sle <8 x i32> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i32> %load.a, <8 x i32> %load.b - store <8 x i32> %sel, <8 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test41: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsd %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp slt <8 x i32> %a, %b + %sel = select <8 x i1> %cmp, <8 x i32> %a, <8 x i32> %b + ret <8 x i32> %sel +} + +define <8 x i32> @test42(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: test42: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm6 +; SSE2-NEXT: pcmpeqd %xmm7, %xmm7 +; SSE2-NEXT: movdqa %xmm6, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm2, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm7 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: pandn %xmm2, %xmm7 +; SSE2-NEXT: por %xmm7, %xmm5 +; SSE2-NEXT: pandn %xmm1, %xmm6 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test42: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsd %xmm2, %xmm0 +; SSE4-NEXT: pminsd %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test42: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsd %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test42: -; AVX2: vpminsd - -; AVX512VL-LABEL: test42: -; AVX512VL: vpminsd -} - -define void @test43(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i32>* - %ptr.b = bitcast i32* %gep.b to <8 x i32>* - %load.a = load <8 x i32>, <8 x i32>* %ptr.a, align 2 - %load.b = load <8 x i32>, <8 x i32>* %ptr.b, align 2 - %cmp = icmp sgt <8 x i32> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i32> %load.a, <8 x i32> %load.b - store <8 x i32> %sel, <8 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test42: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsd %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sle <8 x i32> %a, %b + %sel = select <8 x i1> %cmp, <8 x i32> %a, <8 x i32> %b + ret <8 x i32> %sel +} + +define <8 x i32> @test43(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: test43: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm2, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm0, %xmm5 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm1, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test43: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsd %xmm2, %xmm0 +; SSE4-NEXT: pmaxsd %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test43: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsd %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test43: -; AVX2: vpmaxsd - -; AVX512VL-LABEL: test43: -; AVX512VL: vpmaxsd -} - -define void @test44(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i32>* - %ptr.b = bitcast i32* %gep.b to <8 x i32>* - %load.a = load <8 x i32>, <8 x i32>* %ptr.a, align 2 - %load.b = load <8 x i32>, <8 x i32>* %ptr.b, align 2 - %cmp = icmp sge <8 x i32> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i32> %load.a, <8 x i32> %load.b - store <8 x i32> %sel, <8 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test43: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sgt <8 x i32> %a, %b + %sel = select <8 x i1> %cmp, <8 x i32> %a, <8 x i32> %b + ret <8 x i32> %sel +} + +define <8 x i32> @test44(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: test44: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm6 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm2, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm7 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm7 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm7, %xmm4 +; SSE2-NEXT: pandn %xmm1, %xmm6 +; SSE2-NEXT: pandn %xmm3, %xmm5 +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test44: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsd %xmm2, %xmm0 +; SSE4-NEXT: pmaxsd %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test44: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsd %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test44: -; AVX2: vpmaxsd - -; AVX512VL-LABEL: test44: -; AVX512VL: vpmaxsd -} - -define void @test45(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i32>* - %ptr.b = bitcast i32* %gep.b to <8 x i32>* - %load.a = load <8 x i32>, <8 x i32>* %ptr.a, align 2 - %load.b = load <8 x i32>, <8 x i32>* %ptr.b, align 2 - %cmp = icmp ult <8 x i32> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i32> %load.a, <8 x i32> %load.b - store <8 x i32> %sel, <8 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test44: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sge <8 x i32> %a, %b + %sel = select <8 x i1> %cmp, <8 x i32> %a, <8 x i32> %b + ret <8 x i32> %sel +} + +define <8 x i32> @test45(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: test45: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: pxor %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm4 +; SSE2-NEXT: pand %xmm4, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm4, %xmm0 +; SSE2-NEXT: pand %xmm6, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm6 +; SSE2-NEXT: por %xmm6, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test45: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminud %xmm2, %xmm0 +; SSE4-NEXT: pminud %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test45: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminud %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminud %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test45: -; AVX2: vpminud - -; AVX512VL-LABEL: test45: -; AVX512VL: vpminud -} - -define void @test46(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i32>* - %ptr.b = bitcast i32* %gep.b to <8 x i32>* - %load.a = load <8 x i32>, <8 x i32>* %ptr.a, align 2 - %load.b = load <8 x i32>, <8 x i32>* %ptr.b, align 2 - %cmp = icmp ule <8 x i32> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i32> %load.a, <8 x i32> %load.b - store <8 x i32> %sel, <8 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminud %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test45: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminud %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ult <8 x i32> %a, %b + %sel = select <8 x i1> %cmp, <8 x i32> %a, <8 x i32> %b + ret <8 x i32> %sel +} + +define <8 x i32> @test46(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: test46: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pxor %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm1, %xmm7 +; SSE2-NEXT: pxor %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm7 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm2, %xmm8 +; SSE2-NEXT: pxor %xmm6, %xmm8 +; SSE2-NEXT: pxor %xmm0, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm6 +; SSE2-NEXT: pxor %xmm6, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm6 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm6, %xmm4 +; SSE2-NEXT: pandn %xmm1, %xmm7 +; SSE2-NEXT: pandn %xmm3, %xmm5 +; SSE2-NEXT: por %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test46: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminud %xmm2, %xmm0 +; SSE4-NEXT: pminud %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test46: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminud %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminud %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test46: -; AVX2: vpminud - -; AVX512VL-LABEL: test46: -; AVX512VL: vpminud -} - -define void @test47(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i32>* - %ptr.b = bitcast i32* %gep.b to <8 x i32>* - %load.a = load <8 x i32>, <8 x i32>* %ptr.a, align 2 - %load.b = load <8 x i32>, <8 x i32>* %ptr.b, align 2 - %cmp = icmp ugt <8 x i32> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i32> %load.a, <8 x i32> %load.b - store <8 x i32> %sel, <8 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminud %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test46: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminud %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ule <8 x i32> %a, %b + %sel = select <8 x i1> %cmp, <8 x i32> %a, <8 x i32> %b + ret <8 x i32> %sel +} + +define <8 x i32> @test47(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: test47: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pxor %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm2, %xmm6 +; SSE2-NEXT: pxor %xmm5, %xmm6 +; SSE2-NEXT: pxor %xmm0, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: pand %xmm4, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm4 +; SSE2-NEXT: por %xmm1, %xmm4 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test47: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxud %xmm2, %xmm0 +; SSE4-NEXT: pmaxud %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test47: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxud %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxud %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test47: -; AVX2: vpmaxud - -; AVX512VL-LABEL: test47: -; AVX512VL: vpmaxud -} - -define void @test48(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i32>* - %ptr.b = bitcast i32* %gep.b to <8 x i32>* - %load.a = load <8 x i32>, <8 x i32>* %ptr.a, align 2 - %load.b = load <8 x i32>, <8 x i32>* %ptr.b, align 2 - %cmp = icmp uge <8 x i32> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i32> %load.a, <8 x i32> %load.b - store <8 x i32> %sel, <8 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxud %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test47: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxud %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ugt <8 x i32> %a, %b + %sel = select <8 x i1> %cmp, <8 x i32> %a, <8 x i32> %b + ret <8 x i32> %sel +} + +define <8 x i32> @test48(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: test48: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm3, %xmm7 +; SSE2-NEXT: pxor %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm7 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm0, %xmm8 +; SSE2-NEXT: pxor %xmm6, %xmm8 +; SSE2-NEXT: pxor %xmm2, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm6 +; SSE2-NEXT: pxor %xmm6, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm6 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm6, %xmm4 +; SSE2-NEXT: pandn %xmm1, %xmm7 +; SSE2-NEXT: pandn %xmm3, %xmm5 +; SSE2-NEXT: por %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test48: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxud %xmm2, %xmm0 +; SSE4-NEXT: pmaxud %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test48: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxud %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxud %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test48: -; AVX2: vpmaxud - -; AVX512VL-LABEL: test48: -; AVX512VL: vpmaxud -} - -define void @test49(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <16 x i8>* - %ptr.b = bitcast i8* %gep.b to <16 x i8>* - %load.a = load <16 x i8>, <16 x i8>* %ptr.a, align 2 - %load.b = load <16 x i8>, <16 x i8>* %ptr.b, align 2 - %cmp = icmp slt <16 x i8> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i8> %load.b, <16 x i8> %load.a - store <16 x i8> %sel, <16 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxud %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test48: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxud %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp uge <8 x i32> %a, %b + %sel = select <8 x i1> %cmp, <8 x i32> %a, <8 x i32> %b + ret <8 x i32> %sel +} + +define <16 x i8> @test49(<16 x i8> %a, <16 x i8> %b) { +; SSE2-LABEL: test49: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtb %xmm0, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test49: -; SSE4: pmaxsb - -; AVX1-LABEL: test49: -; AVX1: vpmaxsb - -; AVX2-LABEL: test49: -; AVX2: vpmaxsb - -; AVX512VL-LABEL: test49: -; AVX512VL: vpmaxsb -} - -define void @test50(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <16 x i8>* - %ptr.b = bitcast i8* %gep.b to <16 x i8>* - %load.a = load <16 x i8>, <16 x i8>* %ptr.a, align 2 - %load.b = load <16 x i8>, <16 x i8>* %ptr.b, align 2 - %cmp = icmp sle <16 x i8> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i8> %load.b, <16 x i8> %load.a - store <16 x i8> %sel, <16 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsb %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test49: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp slt <16 x i8> %a, %b + %sel = select <16 x i1> %cmp, <16 x i8> %b, <16 x i8> %a + ret <16 x i8> %sel +} + +define <16 x i8> @test50(<16 x i8> %a, <16 x i8> %b) { +; SSE2-LABEL: test50: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm2 +; SSE2-NEXT: pcmpeqd %xmm3, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test50: -; SSE4: pmaxsb - -; AVX1-LABEL: test50: -; AVX1: vpmaxsb - -; AVX2-LABEL: test50: -; AVX2: vpmaxsb - -; AVX512VL-LABEL: test50: -; AVX512VL: vpmaxsb -} - -define void @test51(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <16 x i8>* - %ptr.b = bitcast i8* %gep.b to <16 x i8>* - %load.a = load <16 x i8>, <16 x i8>* %ptr.a, align 2 - %load.b = load <16 x i8>, <16 x i8>* %ptr.b, align 2 - %cmp = icmp sgt <16 x i8> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i8> %load.b, <16 x i8> %load.a - store <16 x i8> %sel, <16 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsb %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test50: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sle <16 x i8> %a, %b + %sel = select <16 x i1> %cmp, <16 x i8> %b, <16 x i8> %a + ret <16 x i8> %sel +} + +define <16 x i8> @test51(<16 x i8> %a, <16 x i8> %b) { +; SSE2-LABEL: test51: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test51: -; SSE4: pminsb - -; AVX1-LABEL: test51: -; AVX1: vpminsb - -; AVX2-LABEL: test51: -; AVX2: vpminsb - -; AVX512VL-LABEL: test51: -; AVX512VL: vpminsb -} - -define void @test52(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <16 x i8>* - %ptr.b = bitcast i8* %gep.b to <16 x i8>* - %load.a = load <16 x i8>, <16 x i8>* %ptr.a, align 2 - %load.b = load <16 x i8>, <16 x i8>* %ptr.b, align 2 - %cmp = icmp sge <16 x i8> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i8> %load.b, <16 x i8> %load.a - store <16 x i8> %sel, <16 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsb %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test51: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminsb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sgt <16 x i8> %a, %b + %sel = select <16 x i1> %cmp, <16 x i8> %b, <16 x i8> %a + ret <16 x i8> %sel +} + +define <16 x i8> @test52(<16 x i8> %a, <16 x i8> %b) { +; SSE2-LABEL: test52: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pcmpgtb %xmm0, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test52: -; SSE4: pminsb - -; AVX1-LABEL: test52: -; AVX1: vpminsb - -; AVX2-LABEL: test52: -; AVX2: vpminsb - -; AVX512VL-LABEL: test52: -; AVX512VL: vpminsb -} - -define void @test53(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <16 x i8>* - %ptr.b = bitcast i8* %gep.b to <16 x i8>* - %load.a = load <16 x i8>, <16 x i8>* %ptr.a, align 2 - %load.b = load <16 x i8>, <16 x i8>* %ptr.b, align 2 - %cmp = icmp ult <16 x i8> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i8> %load.b, <16 x i8> %load.a - store <16 x i8> %sel, <16 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; SSE2-LABEL: test53: -; SSE2: pmaxub - -; AVX1-LABEL: test53: -; AVX1: vpmaxub - -; AVX2-LABEL: test53: -; AVX2: vpmaxub - -; AVX512VL-LABEL: test53: -; AVX512VL: vpmaxub -} - -define void @test54(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <16 x i8>* - %ptr.b = bitcast i8* %gep.b to <16 x i8>* - %load.a = load <16 x i8>, <16 x i8>* %ptr.a, align 2 - %load.b = load <16 x i8>, <16 x i8>* %ptr.b, align 2 - %cmp = icmp ule <16 x i8> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i8> %load.b, <16 x i8> %load.a - store <16 x i8> %sel, <16 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; SSE2-LABEL: test54: -; SSE2: pmaxub - -; AVX1-LABEL: test54: -; AVX1: vpmaxub - -; AVX2-LABEL: test54: -; AVX2: vpmaxub - -; AVX512VL-LABEL: test54: -; AVX512VL: vpmaxub -} - -define void @test55(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <16 x i8>* - %ptr.b = bitcast i8* %gep.b to <16 x i8>* - %load.a = load <16 x i8>, <16 x i8>* %ptr.a, align 2 - %load.b = load <16 x i8>, <16 x i8>* %ptr.b, align 2 - %cmp = icmp ugt <16 x i8> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i8> %load.b, <16 x i8> %load.a - store <16 x i8> %sel, <16 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; SSE2-LABEL: test55: -; SSE2: pminub - -; AVX1-LABEL: test55: -; AVX1: vpminub - -; AVX2-LABEL: test55: -; AVX2: vpminub - -; AVX512VL-LABEL: test55: -; AVX512VL: vpminub -} - -define void @test56(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <16 x i8>* - %ptr.b = bitcast i8* %gep.b to <16 x i8>* - %load.a = load <16 x i8>, <16 x i8>* %ptr.a, align 2 - %load.b = load <16 x i8>, <16 x i8>* %ptr.b, align 2 - %cmp = icmp uge <16 x i8> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i8> %load.b, <16 x i8> %load.a - store <16 x i8> %sel, <16 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; SSE2-LABEL: test56: -; SSE2: pminub - -; AVX1-LABEL: test56: -; AVX1: vpminub - -; AVX2-LABEL: test56: -; AVX2: vpminub - -; AVX512VL-LABEL: test56: -; AVX512VL: vpminub -} - -define void @test57(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <8 x i16>* - %ptr.b = bitcast i16* %gep.b to <8 x i16>* - %load.a = load <8 x i16>, <8 x i16>* %ptr.a, align 2 - %load.b = load <8 x i16>, <8 x i16>* %ptr.b, align 2 - %cmp = icmp slt <8 x i16> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i16> %load.b, <8 x i16> %load.a - store <8 x i16> %sel, <8 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; SSE2-LABEL: test57: -; SSE2: pmaxsw - -; AVX1-LABEL: test57: -; AVX1: vpmaxsw - -; AVX2-LABEL: test57: -; AVX2: vpmaxsw - -; AVX512VL-LABEL: test57: -; AVX512VL: vpmaxsw -} - -define void @test58(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <8 x i16>* - %ptr.b = bitcast i16* %gep.b to <8 x i16>* - %load.a = load <8 x i16>, <8 x i16>* %ptr.a, align 2 - %load.b = load <8 x i16>, <8 x i16>* %ptr.b, align 2 - %cmp = icmp sle <8 x i16> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i16> %load.b, <8 x i16> %load.a - store <8 x i16> %sel, <8 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; SSE2-LABEL: test58: -; SSE2: pmaxsw - -; AVX1-LABEL: test58: -; AVX1: vpmaxsw - -; AVX2-LABEL: test58: -; AVX2: vpmaxsw - -; AVX512VL-LABEL: test58: -; AVX512VL: vpmaxsw -} - -define void @test59(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <8 x i16>* - %ptr.b = bitcast i16* %gep.b to <8 x i16>* - %load.a = load <8 x i16>, <8 x i16>* %ptr.a, align 2 - %load.b = load <8 x i16>, <8 x i16>* %ptr.b, align 2 - %cmp = icmp sgt <8 x i16> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i16> %load.b, <8 x i16> %load.a - store <8 x i16> %sel, <8 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; SSE2-LABEL: test59: -; SSE2: pminsw - -; AVX1-LABEL: test59: -; AVX1: vpminsw - -; AVX2-LABEL: test59: -; AVX2: vpminsw - -; AVX512VL-LABEL: test59: -; AVX512VL: vpminsw -} - -define void @test60(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <8 x i16>* - %ptr.b = bitcast i16* %gep.b to <8 x i16>* - %load.a = load <8 x i16>, <8 x i16>* %ptr.a, align 2 - %load.b = load <8 x i16>, <8 x i16>* %ptr.b, align 2 - %cmp = icmp sge <8 x i16> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i16> %load.b, <8 x i16> %load.a - store <8 x i16> %sel, <8 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; SSE2-LABEL: test60: -; SSE2: pminsw - -; AVX1-LABEL: test60: -; AVX1: vpminsw - -; AVX2-LABEL: test60: -; AVX2: vpminsw - -; AVX512VL-LABEL: test60: -; AVX512VL: vpminsw -} - -define void @test61(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <8 x i16>* - %ptr.b = bitcast i16* %gep.b to <8 x i16>* - %load.a = load <8 x i16>, <8 x i16>* %ptr.a, align 2 - %load.b = load <8 x i16>, <8 x i16>* %ptr.b, align 2 - %cmp = icmp ult <8 x i16> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i16> %load.b, <8 x i16> %load.a - store <8 x i16> %sel, <8 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsb %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test52: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminsb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sge <16 x i8> %a, %b + %sel = select <16 x i1> %cmp, <16 x i8> %b, <16 x i8> %a + ret <16 x i8> %sel +} + +define <16 x i8> @test53(<16 x i8> %a, <16 x i8> %b) { +; SSE-LABEL: test53: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxub %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test53: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxub %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ult <16 x i8> %a, %b + %sel = select <16 x i1> %cmp, <16 x i8> %b, <16 x i8> %a + ret <16 x i8> %sel +} + +define <16 x i8> @test54(<16 x i8> %a, <16 x i8> %b) { +; SSE-LABEL: test54: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxub %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test54: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxub %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ule <16 x i8> %a, %b + %sel = select <16 x i1> %cmp, <16 x i8> %b, <16 x i8> %a + ret <16 x i8> %sel +} + +define <16 x i8> @test55(<16 x i8> %a, <16 x i8> %b) { +; SSE-LABEL: test55: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminub %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test55: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminub %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ugt <16 x i8> %a, %b + %sel = select <16 x i1> %cmp, <16 x i8> %b, <16 x i8> %a + ret <16 x i8> %sel +} + +define <16 x i8> @test56(<16 x i8> %a, <16 x i8> %b) { +; SSE-LABEL: test56: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminub %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test56: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminub %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp uge <16 x i8> %a, %b + %sel = select <16 x i1> %cmp, <16 x i8> %b, <16 x i8> %a + ret <16 x i8> %sel +} + +define <8 x i16> @test57(<8 x i16> %a, <8 x i16> %b) { +; SSE-LABEL: test57: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxsw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test57: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp slt <8 x i16> %a, %b + %sel = select <8 x i1> %cmp, <8 x i16> %b, <8 x i16> %a + ret <8 x i16> %sel +} + +define <8 x i16> @test58(<8 x i16> %a, <8 x i16> %b) { +; SSE-LABEL: test58: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxsw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test58: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sle <8 x i16> %a, %b + %sel = select <8 x i1> %cmp, <8 x i16> %b, <8 x i16> %a + ret <8 x i16> %sel +} + +define <8 x i16> @test59(<8 x i16> %a, <8 x i16> %b) { +; SSE-LABEL: test59: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminsw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test59: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminsw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sgt <8 x i16> %a, %b + %sel = select <8 x i1> %cmp, <8 x i16> %b, <8 x i16> %a + ret <8 x i16> %sel +} + +define <8 x i16> @test60(<8 x i16> %a, <8 x i16> %b) { +; SSE-LABEL: test60: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminsw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test60: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminsw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sge <8 x i16> %a, %b + %sel = select <8 x i1> %cmp, <8 x i16> %b, <8 x i16> %a + ret <8 x i16> %sel +} + +define <8 x i16> @test61(<8 x i16> %a, <8 x i16> %b) { +; SSE2-LABEL: test61: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtw %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test61: -; SSE4: pmaxuw - -; AVX1-LABEL: test61: -; AVX1: vpmaxuw - -; AVX2-LABEL: test61: -; AVX2: vpmaxuw - -; AVX512VL-LABEL: test61: -; AVX512VL: vpmaxuw -} - -define void @test62(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <8 x i16>* - %ptr.b = bitcast i16* %gep.b to <8 x i16>* - %load.a = load <8 x i16>, <8 x i16>* %ptr.a, align 2 - %load.b = load <8 x i16>, <8 x i16>* %ptr.b, align 2 - %cmp = icmp ule <8 x i16> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i16> %load.b, <8 x i16> %load.a - store <8 x i16> %sel, <8 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxuw %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test61: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxuw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ult <8 x i16> %a, %b + %sel = select <8 x i1> %cmp, <8 x i16> %b, <8 x i16> %a + ret <8 x i16> %sel +} + +define <8 x i16> @test62(<8 x i16> %a, <8 x i16> %b) { +; SSE2-LABEL: test62: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: psubusw %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: pcmpeqw %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test62: -; SSE4: pmaxuw - -; AVX1-LABEL: test62: -; AVX1: vpmaxuw - -; AVX2-LABEL: test62: -; AVX2: vpmaxuw - -; AVX512VL-LABEL: test62: -; AVX512VL: vpmaxuw -} - -define void @test63(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <8 x i16>* - %ptr.b = bitcast i16* %gep.b to <8 x i16>* - %load.a = load <8 x i16>, <8 x i16>* %ptr.a, align 2 - %load.b = load <8 x i16>, <8 x i16>* %ptr.b, align 2 - %cmp = icmp ugt <8 x i16> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i16> %load.b, <8 x i16> %load.a - store <8 x i16> %sel, <8 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxuw %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test62: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxuw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ule <8 x i16> %a, %b + %sel = select <8 x i1> %cmp, <8 x i16> %b, <8 x i16> %a + ret <8 x i16> %sel +} + +define <8 x i16> @test63(<8 x i16> %a, <8 x i16> %b) { +; SSE2-LABEL: test63: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtw %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test63: -; SSE4: pminuw - -; AVX1-LABEL: test63: -; AVX1: vpminuw - -; AVX2-LABEL: test63: -; AVX2: vpminuw - -; AVX512VL-LABEL: test63: -; AVX512VL: vpminuw -} - -define void @test64(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <8 x i16>* - %ptr.b = bitcast i16* %gep.b to <8 x i16>* - %load.a = load <8 x i16>, <8 x i16>* %ptr.a, align 2 - %load.b = load <8 x i16>, <8 x i16>* %ptr.b, align 2 - %cmp = icmp uge <8 x i16> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i16> %load.b, <8 x i16> %load.a - store <8 x i16> %sel, <8 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminuw %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test63: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminuw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ugt <8 x i16> %a, %b + %sel = select <8 x i1> %cmp, <8 x i16> %b, <8 x i16> %a + ret <8 x i16> %sel +} + +define <8 x i16> @test64(<8 x i16> %a, <8 x i16> %b) { +; SSE2-LABEL: test64: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: psubusw %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: pcmpeqw %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test64: -; SSE4: pminuw - -; AVX1-LABEL: test64: -; AVX1: vpminuw - -; AVX2-LABEL: test64: -; AVX2: vpminuw - -; AVX512VL-LABEL: test64: -; AVX512VL: vpminuw -} - -define void @test65(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i32>* - %ptr.b = bitcast i32* %gep.b to <4 x i32>* - %load.a = load <4 x i32>, <4 x i32>* %ptr.a, align 2 - %load.b = load <4 x i32>, <4 x i32>* %ptr.b, align 2 - %cmp = icmp slt <4 x i32> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i32> %load.b, <4 x i32> %load.a - store <4 x i32> %sel, <4 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 4 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminuw %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test64: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminuw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp uge <8 x i16> %a, %b + %sel = select <8 x i1> %cmp, <8 x i16> %b, <8 x i16> %a + ret <8 x i16> %sel +} + +define <4 x i32> @test65(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: test65: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test65: -; SSE4: pmaxsd - -; AVX1-LABEL: test65: -; AVX1: vpmaxsd - -; AVX2-LABEL: test65: -; AVX2: vpmaxsd - -; AVX512VL-LABEL: test65: -; AVX512VL: vpmaxsd -} - -define void @test66(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i32>* - %ptr.b = bitcast i32* %gep.b to <4 x i32>* - %load.a = load <4 x i32>, <4 x i32>* %ptr.a, align 2 - %load.b = load <4 x i32>, <4 x i32>* %ptr.b, align 2 - %cmp = icmp sle <4 x i32> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i32> %load.b, <4 x i32> %load.a - store <4 x i32> %sel, <4 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 4 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsd %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test65: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp slt <4 x i32> %a, %b + %sel = select <4 x i1> %cmp, <4 x i32> %b, <4 x i32> %a + ret <4 x i32> %sel +} + +define <4 x i32> @test66(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: test66: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE2-NEXT: pcmpeqd %xmm3, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test66: -; SSE4: pmaxsd - -; AVX1-LABEL: test66: -; AVX1: vpmaxsd - -; AVX2-LABEL: test66: -; AVX2: vpmaxsd - -; AVX512VL-LABEL: test66: -; AVX512VL: vpmaxsd -} - -define void @test67(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i32>* - %ptr.b = bitcast i32* %gep.b to <4 x i32>* - %load.a = load <4 x i32>, <4 x i32>* %ptr.a, align 2 - %load.b = load <4 x i32>, <4 x i32>* %ptr.b, align 2 - %cmp = icmp sgt <4 x i32> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i32> %load.b, <4 x i32> %load.a - store <4 x i32> %sel, <4 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 4 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsd %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test66: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sle <4 x i32> %a, %b + %sel = select <4 x i1> %cmp, <4 x i32> %b, <4 x i32> %a + ret <4 x i32> %sel +} + +define <4 x i32> @test67(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: test67: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test67: -; SSE4: pminsd - -; AVX1-LABEL: test67: -; AVX1: vpminsd - -; AVX2-LABEL: test67: -; AVX2: vpminsd - -; AVX512VL-LABEL: test67: -; AVX512VL: vpminsd -} - -define void @test68(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i32>* - %ptr.b = bitcast i32* %gep.b to <4 x i32>* - %load.a = load <4 x i32>, <4 x i32>* %ptr.a, align 2 - %load.b = load <4 x i32>, <4 x i32>* %ptr.b, align 2 - %cmp = icmp sge <4 x i32> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i32> %load.b, <4 x i32> %load.a - store <4 x i32> %sel, <4 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 4 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsd %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test67: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminsd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sgt <4 x i32> %a, %b + %sel = select <4 x i1> %cmp, <4 x i32> %b, <4 x i32> %a + ret <4 x i32> %sel +} + +define <4 x i32> @test68(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: test68: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test68: -; SSE4: pminsd - -; AVX1-LABEL: test68: -; AVX1: vpminsd - -; AVX2-LABEL: test68: -; AVX2: vpminsd - -; AVX512VL-LABEL: test68: -; AVX512VL: vpminsd -} - -define void @test69(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i32>* - %ptr.b = bitcast i32* %gep.b to <4 x i32>* - %load.a = load <4 x i32>, <4 x i32>* %ptr.a, align 2 - %load.b = load <4 x i32>, <4 x i32>* %ptr.b, align 2 - %cmp = icmp ult <4 x i32> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i32> %load.b, <4 x i32> %load.a - store <4 x i32> %sel, <4 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 4 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsd %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test68: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminsd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp sge <4 x i32> %a, %b + %sel = select <4 x i1> %cmp, <4 x i32> %b, <4 x i32> %a + ret <4 x i32> %sel +} + +define <4 x i32> @test69(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: test69: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test69: -; SSE4: pmaxud - -; AVX1-LABEL: test69: -; AVX1: vpmaxud - -; AVX2-LABEL: test69: -; AVX2: vpmaxud - -; AVX512VL-LABEL: test69: -; AVX512VL: vpmaxud -} - -define void @test70(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i32>* - %ptr.b = bitcast i32* %gep.b to <4 x i32>* - %load.a = load <4 x i32>, <4 x i32>* %ptr.a, align 2 - %load.b = load <4 x i32>, <4 x i32>* %ptr.b, align 2 - %cmp = icmp ule <4 x i32> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i32> %load.b, <4 x i32> %load.a - store <4 x i32> %sel, <4 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 4 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxud %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test69: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxud %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ult <4 x i32> %a, %b + %sel = select <4 x i1> %cmp, <4 x i32> %b, <4 x i32> %a + ret <4 x i32> %sel +} + +define <4 x i32> @test70(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: test70: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pxor %xmm0, %xmm3 +; SSE2-NEXT: pcmpgtd %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test70: -; SSE4: pmaxud - -; AVX1-LABEL: test70: -; AVX1: vpmaxud - -; AVX2-LABEL: test70: -; AVX2: vpmaxud - -; AVX512VL-LABEL: test70: -; AVX512VL: vpmaxud -} - -define void @test71(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i32>* - %ptr.b = bitcast i32* %gep.b to <4 x i32>* - %load.a = load <4 x i32>, <4 x i32>* %ptr.a, align 2 - %load.b = load <4 x i32>, <4 x i32>* %ptr.b, align 2 - %cmp = icmp ugt <4 x i32> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i32> %load.b, <4 x i32> %load.a - store <4 x i32> %sel, <4 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 4 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxud %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test70: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpmaxud %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ule <4 x i32> %a, %b + %sel = select <4 x i1> %cmp, <4 x i32> %b, <4 x i32> %a + ret <4 x i32> %sel +} + +define <4 x i32> @test71(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: test71: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test71: -; SSE4: pminud - -; AVX1-LABEL: test71: -; AVX1: vpminud - -; AVX2-LABEL: test71: -; AVX2: vpminud - -; AVX512VL-LABEL: test71: -; AVX512VL: vpminud -} - -define void @test72(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i32>* - %ptr.b = bitcast i32* %gep.b to <4 x i32>* - %load.a = load <4 x i32>, <4 x i32>* %ptr.a, align 2 - %load.b = load <4 x i32>, <4 x i32>* %ptr.b, align 2 - %cmp = icmp uge <4 x i32> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i32> %load.b, <4 x i32> %load.a - store <4 x i32> %sel, <4 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 4 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminud %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test71: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminud %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp ugt <4 x i32> %a, %b + %sel = select <4 x i1> %cmp, <4 x i32> %b, <4 x i32> %a + ret <4 x i32> %sel +} + +define <4 x i32> @test72(<4 x i32> %a, <4 x i32> %b) { +; SSE2-LABEL: test72: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pxor %xmm1, %xmm3 +; SSE2-NEXT: pcmpgtd %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; ; SSE4-LABEL: test72: -; SSE4: pminud - -; AVX1-LABEL: test72: -; AVX1: vpminud - -; AVX2-LABEL: test72: -; AVX2: vpminud - -; AVX512VL-LABEL: test72: -; AVX512VL: vpminud -} - -define void @test73(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <32 x i8>* - %ptr.b = bitcast i8* %gep.b to <32 x i8>* - %load.a = load <32 x i8>, <32 x i8>* %ptr.a, align 2 - %load.b = load <32 x i8>, <32 x i8>* %ptr.b, align 2 - %cmp = icmp slt <32 x i8> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i8> %load.b, <32 x i8> %load.a - store <32 x i8> %sel, <32 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminud %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX-LABEL: test72: +; AVX: # BB#0: # %entry +; AVX-NEXT: vpminud %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +entry: + %cmp = icmp uge <4 x i32> %a, %b + %sel = select <4 x i1> %cmp, <4 x i32> %b, <4 x i32> %a + ret <4 x i32> %sel +} + +define <32 x i8> @test73(<32 x i8> %a, <32 x i8> %b) { +; SSE2-LABEL: test73: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm4 +; SSE2-NEXT: movdqa %xmm2, %xmm5 +; SSE2-NEXT: pcmpgtb %xmm0, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: por %xmm2, %xmm5 +; SSE2-NEXT: pand %xmm4, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm4 +; SSE2-NEXT: por %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test73: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsb %xmm2, %xmm0 +; SSE4-NEXT: pmaxsb %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test73: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsb %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test73: -; AVX2: vpmaxsb - -; AVX512VL-LABEL: test73: -; AVX512VL: vpmaxsb -} - -define void @test74(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <32 x i8>* - %ptr.b = bitcast i8* %gep.b to <32 x i8>* - %load.a = load <32 x i8>, <32 x i8>* %ptr.a, align 2 - %load.b = load <32 x i8>, <32 x i8>* %ptr.b, align 2 - %cmp = icmp sle <32 x i8> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i8> %load.b, <32 x i8> %load.a - store <32 x i8> %sel, <32 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test73: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp slt <32 x i8> %a, %b + %sel = select <32 x i1> %cmp, <32 x i8> %b, <32 x i8> %a + ret <32 x i8> %sel +} + +define <32 x i8> @test74(<32 x i8> %a, <32 x i8> %b) { +; SSE2-LABEL: test74: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm6 +; SSE2-NEXT: pcmpgtb %xmm3, %xmm6 +; SSE2-NEXT: pcmpeqd %xmm7, %xmm7 +; SSE2-NEXT: movdqa %xmm6, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pcmpgtb %xmm2, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm7 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: pandn %xmm0, %xmm7 +; SSE2-NEXT: por %xmm7, %xmm5 +; SSE2-NEXT: pandn %xmm3, %xmm6 +; SSE2-NEXT: pandn %xmm1, %xmm4 +; SSE2-NEXT: por %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test74: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsb %xmm2, %xmm0 +; SSE4-NEXT: pmaxsb %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test74: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsb %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test74: -; AVX2: vpmaxsb - -; AVX512VL-LABEL: test74: -; AVX512VL: vpmaxsb -} - -define void @test75(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <32 x i8>* - %ptr.b = bitcast i8* %gep.b to <32 x i8>* - %load.a = load <32 x i8>, <32 x i8>* %ptr.a, align 2 - %load.b = load <32 x i8>, <32 x i8>* %ptr.b, align 2 - %cmp = icmp sgt <32 x i8> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i8> %load.b, <32 x i8> %load.a - store <32 x i8> %sel, <32 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test74: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sle <32 x i8> %a, %b + %sel = select <32 x i1> %cmp, <32 x i8> %b, <32 x i8> %a + ret <32 x i8> %sel +} + +define <32 x i8> @test75(<32 x i8> %a, <32 x i8> %b) { +; SSE2-LABEL: test75: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pcmpgtb %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pcmpgtb %xmm2, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: por %xmm2, %xmm5 +; SSE2-NEXT: pand %xmm4, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm4 +; SSE2-NEXT: por %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test75: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsb %xmm2, %xmm0 +; SSE4-NEXT: pminsb %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test75: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsb %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test75: -; AVX2: vpminsb - -; AVX512VL-LABEL: test75: -; AVX512VL: vpminsb -} - -define void @test76(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <32 x i8>* - %ptr.b = bitcast i8* %gep.b to <32 x i8>* - %load.a = load <32 x i8>, <32 x i8>* %ptr.a, align 2 - %load.b = load <32 x i8>, <32 x i8>* %ptr.b, align 2 - %cmp = icmp sge <32 x i8> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i8> %load.b, <32 x i8> %load.a - store <32 x i8> %sel, <32 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsb %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test75: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsb %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sgt <32 x i8> %a, %b + %sel = select <32 x i1> %cmp, <32 x i8> %b, <32 x i8> %a + ret <32 x i8> %sel +} + +define <32 x i8> @test76(<32 x i8> %a, <32 x i8> %b) { +; SSE2-LABEL: test76: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm6 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm2, %xmm7 +; SSE2-NEXT: pcmpgtb %xmm0, %xmm7 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: pandn %xmm2, %xmm7 +; SSE2-NEXT: pandn %xmm0, %xmm4 +; SSE2-NEXT: por %xmm7, %xmm4 +; SSE2-NEXT: pandn %xmm3, %xmm6 +; SSE2-NEXT: pandn %xmm1, %xmm5 +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test76: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsb %xmm2, %xmm0 +; SSE4-NEXT: pminsb %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test76: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsb %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test76: -; AVX2: vpminsb - -; AVX512VL-LABEL: test76: -; AVX512VL: vpminsb -} - -define void @test77(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <32 x i8>* - %ptr.b = bitcast i8* %gep.b to <32 x i8>* - %load.a = load <32 x i8>, <32 x i8>* %ptr.a, align 2 - %load.b = load <32 x i8>, <32 x i8>* %ptr.b, align 2 - %cmp = icmp ult <32 x i8> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i8> %load.b, <32 x i8> %load.a - store <32 x i8> %sel, <32 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsb %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test76: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsb %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sge <32 x i8> %a, %b + %sel = select <32 x i1> %cmp, <32 x i8> %b, <32 x i8> %a + ret <32 x i8> %sel +} + +define <32 x i8> @test77(<32 x i8> %a, <32 x i8> %b) { +; SSE-LABEL: test77: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxub %xmm2, %xmm0 +; SSE-NEXT: pmaxub %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: test77: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxub %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxub %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test77: -; AVX2: vpmaxub - -; AVX512VL-LABEL: test77: -; AVX512VL: vpmaxub -} - -define void @test78(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <32 x i8>* - %ptr.b = bitcast i8* %gep.b to <32 x i8>* - %load.a = load <32 x i8>, <32 x i8>* %ptr.a, align 2 - %load.b = load <32 x i8>, <32 x i8>* %ptr.b, align 2 - %cmp = icmp ule <32 x i8> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i8> %load.b, <32 x i8> %load.a - store <32 x i8> %sel, <32 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxub %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test77: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxub %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ult <32 x i8> %a, %b + %sel = select <32 x i1> %cmp, <32 x i8> %b, <32 x i8> %a + ret <32 x i8> %sel +} + +define <32 x i8> @test78(<32 x i8> %a, <32 x i8> %b) { +; SSE-LABEL: test78: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxub %xmm2, %xmm0 +; SSE-NEXT: pmaxub %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: test78: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxub %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxub %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test78: -; AVX2: vpmaxub - -; AVX512VL-LABEL: test78: -; AVX512VL: vpmaxub -} - -define void @test79(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <32 x i8>* - %ptr.b = bitcast i8* %gep.b to <32 x i8>* - %load.a = load <32 x i8>, <32 x i8>* %ptr.a, align 2 - %load.b = load <32 x i8>, <32 x i8>* %ptr.b, align 2 - %cmp = icmp ugt <32 x i8> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i8> %load.b, <32 x i8> %load.a - store <32 x i8> %sel, <32 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxub %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test78: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxub %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ule <32 x i8> %a, %b + %sel = select <32 x i1> %cmp, <32 x i8> %b, <32 x i8> %a + ret <32 x i8> %sel +} + +define <32 x i8> @test79(<32 x i8> %a, <32 x i8> %b) { +; SSE-LABEL: test79: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminub %xmm2, %xmm0 +; SSE-NEXT: pminub %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: test79: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminub %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminub %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test79: -; AVX2: vpminub - -; AVX512VL-LABEL: test79: -; AVX512VL: vpminub -} - -define void @test80(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <32 x i8>* - %ptr.b = bitcast i8* %gep.b to <32 x i8>* - %load.a = load <32 x i8>, <32 x i8>* %ptr.a, align 2 - %load.b = load <32 x i8>, <32 x i8>* %ptr.b, align 2 - %cmp = icmp uge <32 x i8> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i8> %load.b, <32 x i8> %load.a - store <32 x i8> %sel, <32 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminub %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test79: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminub %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ugt <32 x i8> %a, %b + %sel = select <32 x i1> %cmp, <32 x i8> %b, <32 x i8> %a + ret <32 x i8> %sel +} + +define <32 x i8> @test80(<32 x i8> %a, <32 x i8> %b) { +; SSE-LABEL: test80: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminub %xmm2, %xmm0 +; SSE-NEXT: pminub %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: test80: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminub %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminub %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test80: -; AVX2: vpminub - -; AVX512VL-LABEL: test80: -; AVX512VL: vpminub -} - -define void @test81(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <16 x i16>* - %ptr.b = bitcast i16* %gep.b to <16 x i16>* - %load.a = load <16 x i16>, <16 x i16>* %ptr.a, align 2 - %load.b = load <16 x i16>, <16 x i16>* %ptr.b, align 2 - %cmp = icmp slt <16 x i16> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i16> %load.b, <16 x i16> %load.a - store <16 x i16> %sel, <16 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminub %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test80: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminub %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp uge <32 x i8> %a, %b + %sel = select <32 x i1> %cmp, <32 x i8> %b, <32 x i8> %a + ret <32 x i8> %sel +} + +define <16 x i16> @test81(<16 x i16> %a, <16 x i16> %b) { +; SSE-LABEL: test81: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxsw %xmm2, %xmm0 +; SSE-NEXT: pmaxsw %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: test81: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test81: -; AVX2: vpmaxsw - -; AVX512VL-LABEL: test81: -; AVX512VL: vpmaxsw -} - -define void @test82(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <16 x i16>* - %ptr.b = bitcast i16* %gep.b to <16 x i16>* - %load.a = load <16 x i16>, <16 x i16>* %ptr.a, align 2 - %load.b = load <16 x i16>, <16 x i16>* %ptr.b, align 2 - %cmp = icmp sle <16 x i16> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i16> %load.b, <16 x i16> %load.a - store <16 x i16> %sel, <16 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test81: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp slt <16 x i16> %a, %b + %sel = select <16 x i1> %cmp, <16 x i16> %b, <16 x i16> %a + ret <16 x i16> %sel +} + +define <16 x i16> @test82(<16 x i16> %a, <16 x i16> %b) { +; SSE-LABEL: test82: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxsw %xmm2, %xmm0 +; SSE-NEXT: pmaxsw %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: test82: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test82: -; AVX2: vpmaxsw - -; AVX512VL-LABEL: test82: -; AVX512VL: vpmaxsw -} - -define void @test83(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <16 x i16>* - %ptr.b = bitcast i16* %gep.b to <16 x i16>* - %load.a = load <16 x i16>, <16 x i16>* %ptr.a, align 2 - %load.b = load <16 x i16>, <16 x i16>* %ptr.b, align 2 - %cmp = icmp sgt <16 x i16> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i16> %load.b, <16 x i16> %load.a - store <16 x i16> %sel, <16 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test82: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sle <16 x i16> %a, %b + %sel = select <16 x i1> %cmp, <16 x i16> %b, <16 x i16> %a + ret <16 x i16> %sel +} + +define <16 x i16> @test83(<16 x i16> %a, <16 x i16> %b) { +; SSE-LABEL: test83: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminsw %xmm2, %xmm0 +; SSE-NEXT: pminsw %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: test83: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test83: -; AVX2: vpminsw - -; AVX512VL-LABEL: test83: -; AVX512VL: vpminsw -} - -define void @test84(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <16 x i16>* - %ptr.b = bitcast i16* %gep.b to <16 x i16>* - %load.a = load <16 x i16>, <16 x i16>* %ptr.a, align 2 - %load.b = load <16 x i16>, <16 x i16>* %ptr.b, align 2 - %cmp = icmp sge <16 x i16> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i16> %load.b, <16 x i16> %load.a - store <16 x i16> %sel, <16 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test83: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsw %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sgt <16 x i16> %a, %b + %sel = select <16 x i1> %cmp, <16 x i16> %b, <16 x i16> %a + ret <16 x i16> %sel +} + +define <16 x i16> @test84(<16 x i16> %a, <16 x i16> %b) { +; SSE-LABEL: test84: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminsw %xmm2, %xmm0 +; SSE-NEXT: pminsw %xmm3, %xmm1 +; SSE-NEXT: retq +; +; AVX1-LABEL: test84: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test84: -; AVX2: vpminsw - -; AVX512VL-LABEL: test84: -; AVX512VL: vpminsw -} - -define void @test85(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <16 x i16>* - %ptr.b = bitcast i16* %gep.b to <16 x i16>* - %load.a = load <16 x i16>, <16 x i16>* %ptr.a, align 2 - %load.b = load <16 x i16>, <16 x i16>* %ptr.b, align 2 - %cmp = icmp ult <16 x i16> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i16> %load.b, <16 x i16> %load.a - store <16 x i16> %sel, <16 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test84: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsw %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sge <16 x i16> %a, %b + %sel = select <16 x i1> %cmp, <16 x i16> %b, <16 x i16> %a + ret <16 x i16> %sel +} + +define <16 x i16> @test85(<16 x i16> %a, <16 x i16> %b) { +; SSE2-LABEL: test85: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE2-NEXT: movdqa %xmm1, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: movdqa %xmm3, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: pcmpgtw %xmm6, %xmm5 +; SSE2-NEXT: movdqa %xmm0, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: pxor %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtw %xmm6, %xmm4 +; SSE2-NEXT: pand %xmm4, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm4 +; SSE2-NEXT: por %xmm2, %xmm4 +; SSE2-NEXT: pand %xmm5, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm5 +; SSE2-NEXT: por %xmm3, %xmm5 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test85: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxuw %xmm2, %xmm0 +; SSE4-NEXT: pmaxuw %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test85: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxuw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxuw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test85: -; AVX2: vpmaxuw - -; AVX512VL-LABEL: test85: -; AVX512VL: vpmaxuw -} - -define void @test86(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <16 x i16>* - %ptr.b = bitcast i16* %gep.b to <16 x i16>* - %load.a = load <16 x i16>, <16 x i16>* %ptr.a, align 2 - %load.b = load <16 x i16>, <16 x i16>* %ptr.b, align 2 - %cmp = icmp ule <16 x i16> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i16> %load.b, <16 x i16> %load.a - store <16 x i16> %sel, <16 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxuw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test85: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxuw %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ult <16 x i16> %a, %b + %sel = select <16 x i1> %cmp, <16 x i16> %b, <16 x i16> %a + ret <16 x i16> %sel +} + +define <16 x i16> @test86(<16 x i16> %a, <16 x i16> %b) { +; SSE2-LABEL: test86: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: psubusw %xmm3, %xmm4 +; SSE2-NEXT: pxor %xmm6, %xmm6 +; SSE2-NEXT: pcmpeqw %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: psubusw %xmm2, %xmm5 +; SSE2-NEXT: pcmpeqw %xmm6, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: por %xmm2, %xmm5 +; SSE2-NEXT: pand %xmm4, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm4 +; SSE2-NEXT: por %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test86: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxuw %xmm2, %xmm0 +; SSE4-NEXT: pmaxuw %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test86: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxuw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxuw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test86: -; AVX2: vpmaxuw - -; AVX512VL-LABEL: test86: -; AVX512VL: vpmaxuw -} - -define void @test87(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <16 x i16>* - %ptr.b = bitcast i16* %gep.b to <16 x i16>* - %load.a = load <16 x i16>, <16 x i16>* %ptr.a, align 2 - %load.b = load <16 x i16>, <16 x i16>* %ptr.b, align 2 - %cmp = icmp ugt <16 x i16> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i16> %load.b, <16 x i16> %load.a - store <16 x i16> %sel, <16 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxuw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test86: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxuw %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ule <16 x i16> %a, %b + %sel = select <16 x i1> %cmp, <16 x i16> %b, <16 x i16> %a + ret <16 x i16> %sel +} + +define <16 x i16> @test87(<16 x i16> %a, <16 x i16> %b) { +; SSE2-LABEL: test87: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: pcmpgtw %xmm6, %xmm5 +; SSE2-NEXT: movdqa %xmm2, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: pxor %xmm0, %xmm4 +; SSE2-NEXT: pcmpgtw %xmm6, %xmm4 +; SSE2-NEXT: pand %xmm4, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm4 +; SSE2-NEXT: por %xmm2, %xmm4 +; SSE2-NEXT: pand %xmm5, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm5 +; SSE2-NEXT: por %xmm3, %xmm5 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test87: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminuw %xmm2, %xmm0 +; SSE4-NEXT: pminuw %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test87: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminuw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminuw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test87: -; AVX2: vpminuw - -; AVX512VL-LABEL: test87: -; AVX512VL: vpminuw -} - -define void @test88(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <16 x i16>* - %ptr.b = bitcast i16* %gep.b to <16 x i16>* - %load.a = load <16 x i16>, <16 x i16>* %ptr.a, align 2 - %load.b = load <16 x i16>, <16 x i16>* %ptr.b, align 2 - %cmp = icmp uge <16 x i16> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i16> %load.b, <16 x i16> %load.a - store <16 x i16> %sel, <16 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminuw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test87: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminuw %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ugt <16 x i16> %a, %b + %sel = select <16 x i1> %cmp, <16 x i16> %b, <16 x i16> %a + ret <16 x i16> %sel +} + +define <16 x i16> @test88(<16 x i16> %a, <16 x i16> %b) { +; SSE2-LABEL: test88: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: psubusw %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm6, %xmm6 +; SSE2-NEXT: pcmpeqw %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm2, %xmm5 +; SSE2-NEXT: psubusw %xmm0, %xmm5 +; SSE2-NEXT: pcmpeqw %xmm6, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: por %xmm2, %xmm5 +; SSE2-NEXT: pand %xmm4, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm4 +; SSE2-NEXT: por %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test88: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminuw %xmm2, %xmm0 +; SSE4-NEXT: pminuw %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test88: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminuw %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminuw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test88: -; AVX2: vpminuw - -; AVX512VL-LABEL: test88: -; AVX512VL: vpminuw -} - -define void @test89(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i32>* - %ptr.b = bitcast i32* %gep.b to <8 x i32>* - %load.a = load <8 x i32>, <8 x i32>* %ptr.a, align 2 - %load.b = load <8 x i32>, <8 x i32>* %ptr.b, align 2 - %cmp = icmp slt <8 x i32> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i32> %load.b, <8 x i32> %load.a - store <8 x i32> %sel, <8 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminuw %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test88: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminuw %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp uge <16 x i16> %a, %b + %sel = select <16 x i1> %cmp, <16 x i16> %b, <16 x i16> %a + ret <16 x i16> %sel +} + +define <8 x i32> @test89(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: test89: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm4 +; SSE2-NEXT: movdqa %xmm2, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: por %xmm2, %xmm5 +; SSE2-NEXT: pand %xmm4, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm4 +; SSE2-NEXT: por %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test89: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsd %xmm2, %xmm0 +; SSE4-NEXT: pmaxsd %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test89: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsd %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test89: -; AVX2: vpmaxsd - -; AVX512VL-LABEL: test89: -; AVX512VL: vpmaxsd -} - -define void @test90(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i32>* - %ptr.b = bitcast i32* %gep.b to <8 x i32>* - %load.a = load <8 x i32>, <8 x i32>* %ptr.a, align 2 - %load.b = load <8 x i32>, <8 x i32>* %ptr.b, align 2 - %cmp = icmp sle <8 x i32> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i32> %load.b, <8 x i32> %load.a - store <8 x i32> %sel, <8 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test89: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp slt <8 x i32> %a, %b + %sel = select <8 x i1> %cmp, <8 x i32> %b, <8 x i32> %a + ret <8 x i32> %sel +} + +define <8 x i32> @test90(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: test90: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm6 +; SSE2-NEXT: pcmpeqd %xmm7, %xmm7 +; SSE2-NEXT: movdqa %xmm6, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm2, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm7 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: pandn %xmm0, %xmm7 +; SSE2-NEXT: por %xmm7, %xmm5 +; SSE2-NEXT: pandn %xmm3, %xmm6 +; SSE2-NEXT: pandn %xmm1, %xmm4 +; SSE2-NEXT: por %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test90: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsd %xmm2, %xmm0 +; SSE4-NEXT: pmaxsd %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test90: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxsd %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test90: -; AVX2: vpmaxsd - -; AVX512VL-LABEL: test90: -; AVX512VL: vpmaxsd -} - -define void @test91(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i32>* - %ptr.b = bitcast i32* %gep.b to <8 x i32>* - %load.a = load <8 x i32>, <8 x i32>* %ptr.a, align 2 - %load.b = load <8 x i32>, <8 x i32>* %ptr.b, align 2 - %cmp = icmp sgt <8 x i32> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i32> %load.b, <8 x i32> %load.a - store <8 x i32> %sel, <8 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test90: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sle <8 x i32> %a, %b + %sel = select <8 x i1> %cmp, <8 x i32> %b, <8 x i32> %a + ret <8 x i32> %sel +} + +define <8 x i32> @test91(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: test91: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm2, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: por %xmm2, %xmm5 +; SSE2-NEXT: pand %xmm4, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm4 +; SSE2-NEXT: por %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test91: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsd %xmm2, %xmm0 +; SSE4-NEXT: pminsd %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test91: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsd %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test91: -; AVX2: vpminsd - -; AVX512VL-LABEL: test91: -; AVX512VL: vpminsd -} - -define void @test92(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i32>* - %ptr.b = bitcast i32* %gep.b to <8 x i32>* - %load.a = load <8 x i32>, <8 x i32>* %ptr.a, align 2 - %load.b = load <8 x i32>, <8 x i32>* %ptr.b, align 2 - %cmp = icmp sge <8 x i32> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i32> %load.b, <8 x i32> %load.a - store <8 x i32> %sel, <8 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test91: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsd %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sgt <8 x i32> %a, %b + %sel = select <8 x i1> %cmp, <8 x i32> %b, <8 x i32> %a + ret <8 x i32> %sel +} + +define <8 x i32> @test92(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: test92: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm6 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm2, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm7 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: pandn %xmm2, %xmm7 +; SSE2-NEXT: pandn %xmm0, %xmm4 +; SSE2-NEXT: por %xmm7, %xmm4 +; SSE2-NEXT: pandn %xmm3, %xmm6 +; SSE2-NEXT: pandn %xmm1, %xmm5 +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test92: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsd %xmm2, %xmm0 +; SSE4-NEXT: pminsd %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test92: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminsd %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminsd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test92: -; AVX2: vpminsd - -; AVX512VL-LABEL: test92: -; AVX512VL: vpminsd -} - -define void @test93(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i32>* - %ptr.b = bitcast i32* %gep.b to <8 x i32>* - %load.a = load <8 x i32>, <8 x i32>* %ptr.a, align 2 - %load.b = load <8 x i32>, <8 x i32>* %ptr.b, align 2 - %cmp = icmp ult <8 x i32> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i32> %load.b, <8 x i32> %load.a - store <8 x i32> %sel, <8 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test92: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsd %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sge <8 x i32> %a, %b + %sel = select <8 x i1> %cmp, <8 x i32> %b, <8 x i32> %a + ret <8 x i32> %sel +} + +define <8 x i32> @test93(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: test93: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: movdqa %xmm3, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm5 +; SSE2-NEXT: movdqa %xmm0, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: pxor %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm4 +; SSE2-NEXT: pand %xmm4, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm4 +; SSE2-NEXT: por %xmm2, %xmm4 +; SSE2-NEXT: pand %xmm5, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm5 +; SSE2-NEXT: por %xmm3, %xmm5 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test93: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxud %xmm2, %xmm0 +; SSE4-NEXT: pmaxud %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test93: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxud %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxud %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test93: -; AVX2: vpmaxud - -; AVX512VL-LABEL: test93: -; AVX512VL: vpmaxud -} - -define void @test94(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i32>* - %ptr.b = bitcast i32* %gep.b to <8 x i32>* - %load.a = load <8 x i32>, <8 x i32>* %ptr.a, align 2 - %load.b = load <8 x i32>, <8 x i32>* %ptr.b, align 2 - %cmp = icmp ule <8 x i32> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i32> %load.b, <8 x i32> %load.a - store <8 x i32> %sel, <8 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxud %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test93: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxud %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ult <8 x i32> %a, %b + %sel = select <8 x i1> %cmp, <8 x i32> %b, <8 x i32> %a + ret <8 x i32> %sel +} + +define <8 x i32> @test94(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: test94: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pxor %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm1, %xmm7 +; SSE2-NEXT: pxor %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm7 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm2, %xmm8 +; SSE2-NEXT: pxor %xmm6, %xmm8 +; SSE2-NEXT: pxor %xmm0, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm6 +; SSE2-NEXT: pxor %xmm6, %xmm4 +; SSE2-NEXT: pandn %xmm2, %xmm6 +; SSE2-NEXT: pandn %xmm0, %xmm4 +; SSE2-NEXT: por %xmm6, %xmm4 +; SSE2-NEXT: pandn %xmm3, %xmm7 +; SSE2-NEXT: pandn %xmm1, %xmm5 +; SSE2-NEXT: por %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test94: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxud %xmm2, %xmm0 +; SSE4-NEXT: pmaxud %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test94: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpmaxud %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpmaxud %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test94: -; AVX2: vpmaxud - -; AVX512VL-LABEL: test94: -; AVX512VL: vpmaxud -} - -define void @test95(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i32>* - %ptr.b = bitcast i32* %gep.b to <8 x i32>* - %load.a = load <8 x i32>, <8 x i32>* %ptr.a, align 2 - %load.b = load <8 x i32>, <8 x i32>* %ptr.b, align 2 - %cmp = icmp ugt <8 x i32> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i32> %load.b, <8 x i32> %load.a - store <8 x i32> %sel, <8 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxud %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test94: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxud %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ule <8 x i32> %a, %b + %sel = select <8 x i1> %cmp, <8 x i32> %b, <8 x i32> %a + ret <8 x i32> %sel +} + +define <8 x i32> @test95(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: test95: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm5 +; SSE2-NEXT: movdqa %xmm2, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: pxor %xmm0, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm4 +; SSE2-NEXT: pand %xmm4, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm4 +; SSE2-NEXT: por %xmm2, %xmm4 +; SSE2-NEXT: pand %xmm5, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm5 +; SSE2-NEXT: por %xmm3, %xmm5 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test95: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminud %xmm2, %xmm0 +; SSE4-NEXT: pminud %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test95: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminud %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminud %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test95: -; AVX2: vpminud - -; AVX512VL-LABEL: test95: -; AVX512VL: vpminud -} - -define void @test96(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i32>* - %ptr.b = bitcast i32* %gep.b to <8 x i32>* - %load.a = load <8 x i32>, <8 x i32>* %ptr.a, align 2 - %load.b = load <8 x i32>, <8 x i32>* %ptr.b, align 2 - %cmp = icmp uge <8 x i32> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i32> %load.b, <8 x i32> %load.a - store <8 x i32> %sel, <8 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminud %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test95: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminud %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ugt <8 x i32> %a, %b + %sel = select <8 x i1> %cmp, <8 x i32> %b, <8 x i32> %a + ret <8 x i32> %sel +} + +define <8 x i32> @test96(<8 x i32> %a, <8 x i32> %b) { +; SSE2-LABEL: test96: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm3, %xmm7 +; SSE2-NEXT: pxor %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm7 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm0, %xmm8 +; SSE2-NEXT: pxor %xmm6, %xmm8 +; SSE2-NEXT: pxor %xmm2, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm6 +; SSE2-NEXT: pxor %xmm6, %xmm4 +; SSE2-NEXT: pandn %xmm2, %xmm6 +; SSE2-NEXT: pandn %xmm0, %xmm4 +; SSE2-NEXT: por %xmm6, %xmm4 +; SSE2-NEXT: pandn %xmm3, %xmm7 +; SSE2-NEXT: pandn %xmm1, %xmm5 +; SSE2-NEXT: por %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test96: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminud %xmm2, %xmm0 +; SSE4-NEXT: pminud %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test96: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpminud %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpminud %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-NEXT: retq +; ; AVX2-LABEL: test96: -; AVX2: vpminud - -; AVX512VL-LABEL: test96: -; AVX512VL: vpminud +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminud %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512F-LABEL: test96: +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminud %ymm1, %ymm0, %ymm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp uge <8 x i32> %a, %b + %sel = select <8 x i1> %cmp, <8 x i32> %b, <8 x i32> %a + ret <8 x i32> %sel } ; ---------------------------- -define void @test97(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <64 x i8>* - %ptr.b = bitcast i8* %gep.b to <64 x i8>* - %load.a = load <64 x i8>, <64 x i8>* %ptr.a, align 2 - %load.b = load <64 x i8>, <64 x i8>* %ptr.b, align 2 - %cmp = icmp slt <64 x i8> %load.a, %load.b - %sel = select <64 x i1> %cmp, <64 x i8> %load.a, <64 x i8> %load.b - store <64 x i8> %sel, <64 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +define <64 x i8> @test97(<64 x i8> %a, <64 x i8> %b) { +; SSE2-LABEL: test97: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: pcmpgtb %xmm3, %xmm8 +; SSE2-NEXT: movdqa %xmm6, %xmm9 +; SSE2-NEXT: pcmpgtb %xmm2, %xmm9 +; SSE2-NEXT: movdqa %xmm5, %xmm10 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm10 +; SSE2-NEXT: movdqa %xmm4, %xmm11 +; SSE2-NEXT: pcmpgtb %xmm0, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm11 +; SSE2-NEXT: por %xmm11, %xmm0 +; SSE2-NEXT: pand %xmm10, %xmm1 +; SSE2-NEXT: pandn %xmm5, %xmm10 +; SSE2-NEXT: por %xmm10, %xmm1 +; SSE2-NEXT: pand %xmm9, %xmm2 +; SSE2-NEXT: pandn %xmm6, %xmm9 +; SSE2-NEXT: por %xmm9, %xmm2 +; SSE2-NEXT: pand %xmm8, %xmm3 +; SSE2-NEXT: pandn %xmm7, %xmm8 +; SSE2-NEXT: por %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test97: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsb %xmm4, %xmm0 +; SSE4-NEXT: pminsb %xmm5, %xmm1 +; SSE4-NEXT: pminsb %xmm6, %xmm2 +; SSE4-NEXT: pminsb %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test97: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminsb %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminsb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminsb %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminsb %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test97: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsb %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminsb %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test97: -; AVX512BW: vpminsb {{.*}} -} - -define void @test98(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <64 x i8>* - %ptr.b = bitcast i8* %gep.b to <64 x i8>* - %load.a = load <64 x i8>, <64 x i8>* %ptr.a, align 2 - %load.b = load <64 x i8>, <64 x i8>* %ptr.b, align 2 - %cmp = icmp sle <64 x i8> %load.a, %load.b - %sel = select <64 x i1> %cmp, <64 x i8> %load.a, <64 x i8> %load.b - store <64 x i8> %sel, <64 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminsb %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp slt <64 x i8> %a, %b + %sel = select <64 x i1> %cmp, <64 x i8> %a, <64 x i8> %b + ret <64 x i8> %sel +} + +define <64 x i8> @test98(<64 x i8> %a, <64 x i8> %b) { +; SSE2-LABEL: test98: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: movdqa %xmm8, %xmm12 +; SSE2-NEXT: pcmpgtb %xmm7, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm13, %xmm13 +; SSE2-NEXT: movdqa %xmm12, %xmm3 +; SSE2-NEXT: pxor %xmm13, %xmm3 +; SSE2-NEXT: movdqa %xmm9, %xmm14 +; SSE2-NEXT: pcmpgtb %xmm6, %xmm14 +; SSE2-NEXT: movdqa %xmm14, %xmm2 +; SSE2-NEXT: pxor %xmm13, %xmm2 +; SSE2-NEXT: movdqa %xmm1, %xmm15 +; SSE2-NEXT: pcmpgtb %xmm5, %xmm15 +; SSE2-NEXT: movdqa %xmm15, %xmm10 +; SSE2-NEXT: pxor %xmm13, %xmm10 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtb %xmm4, %xmm11 +; SSE2-NEXT: pxor %xmm11, %xmm13 +; SSE2-NEXT: pandn %xmm0, %xmm11 +; SSE2-NEXT: pandn %xmm4, %xmm13 +; SSE2-NEXT: por %xmm13, %xmm11 +; SSE2-NEXT: pandn %xmm1, %xmm15 +; SSE2-NEXT: pandn %xmm5, %xmm10 +; SSE2-NEXT: por %xmm15, %xmm10 +; SSE2-NEXT: pandn %xmm9, %xmm14 +; SSE2-NEXT: pandn %xmm6, %xmm2 +; SSE2-NEXT: por %xmm14, %xmm2 +; SSE2-NEXT: pandn %xmm8, %xmm12 +; SSE2-NEXT: pandn %xmm7, %xmm3 +; SSE2-NEXT: por %xmm12, %xmm3 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test98: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsb %xmm4, %xmm0 +; SSE4-NEXT: pminsb %xmm5, %xmm1 +; SSE4-NEXT: pminsb %xmm6, %xmm2 +; SSE4-NEXT: pminsb %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test98: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminsb %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminsb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminsb %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminsb %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test98: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsb %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminsb %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test98: -; AVX512BW: vpminsb {{.*}} -} - -define void @test99(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <64 x i8>* - %ptr.b = bitcast i8* %gep.b to <64 x i8>* - %load.a = load <64 x i8>, <64 x i8>* %ptr.a, align 2 - %load.b = load <64 x i8>, <64 x i8>* %ptr.b, align 2 - %cmp = icmp sgt <64 x i8> %load.a, %load.b - %sel = select <64 x i1> %cmp, <64 x i8> %load.a, <64 x i8> %load.b - store <64 x i8> %sel, <64 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminsb %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sle <64 x i8> %a, %b + %sel = select <64 x i1> %cmp, <64 x i8> %a, <64 x i8> %b + ret <64 x i8> %sel +} + +define <64 x i8> @test99(<64 x i8> %a, <64 x i8> %b) { +; SSE2-LABEL: test99: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: pcmpgtb %xmm7, %xmm3 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: pcmpgtb %xmm6, %xmm9 +; SSE2-NEXT: movdqa %xmm1, %xmm10 +; SSE2-NEXT: pcmpgtb %xmm5, %xmm10 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtb %xmm4, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm11 +; SSE2-NEXT: por %xmm0, %xmm11 +; SSE2-NEXT: pand %xmm10, %xmm1 +; SSE2-NEXT: pandn %xmm5, %xmm10 +; SSE2-NEXT: por %xmm1, %xmm10 +; SSE2-NEXT: pand %xmm9, %xmm2 +; SSE2-NEXT: pandn %xmm6, %xmm9 +; SSE2-NEXT: por %xmm2, %xmm9 +; SSE2-NEXT: pand %xmm3, %xmm8 +; SSE2-NEXT: pandn %xmm7, %xmm3 +; SSE2-NEXT: por %xmm8, %xmm3 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test99: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsb %xmm4, %xmm0 +; SSE4-NEXT: pmaxsb %xmm5, %xmm1 +; SSE4-NEXT: pmaxsb %xmm6, %xmm2 +; SSE4-NEXT: pmaxsb %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test99: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxsb %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxsb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxsb %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxsb %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test99: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsb %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxsb %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test99: -; AVX512BW: vpmaxsb {{.*}} -} - -define void @test100(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <64 x i8>* - %ptr.b = bitcast i8* %gep.b to <64 x i8>* - %load.a = load <64 x i8>, <64 x i8>* %ptr.a, align 2 - %load.b = load <64 x i8>, <64 x i8>* %ptr.b, align 2 - %cmp = icmp sge <64 x i8> %load.a, %load.b - %sel = select <64 x i1> %cmp, <64 x i8> %load.a, <64 x i8> %load.b - store <64 x i8> %sel, <64 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxsb %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sgt <64 x i8> %a, %b + %sel = select <64 x i1> %cmp, <64 x i8> %a, <64 x i8> %b + ret <64 x i8> %sel +} + +define <64 x i8> @test100(<64 x i8> %a, <64 x i8> %b) { +; SSE2-LABEL: test100: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: movdqa %xmm0, %xmm10 +; SSE2-NEXT: movdqa %xmm7, %xmm12 +; SSE2-NEXT: pcmpgtb %xmm8, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE2-NEXT: movdqa %xmm12, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm3 +; SSE2-NEXT: movdqa %xmm6, %xmm13 +; SSE2-NEXT: pcmpgtb %xmm9, %xmm13 +; SSE2-NEXT: movdqa %xmm13, %xmm2 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm5, %xmm14 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm14 +; SSE2-NEXT: movdqa %xmm14, %xmm11 +; SSE2-NEXT: pxor %xmm0, %xmm11 +; SSE2-NEXT: movdqa %xmm4, %xmm15 +; SSE2-NEXT: pcmpgtb %xmm10, %xmm15 +; SSE2-NEXT: pxor %xmm15, %xmm0 +; SSE2-NEXT: pandn %xmm10, %xmm15 +; SSE2-NEXT: pandn %xmm4, %xmm0 +; SSE2-NEXT: por %xmm15, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm14 +; SSE2-NEXT: pandn %xmm5, %xmm11 +; SSE2-NEXT: por %xmm14, %xmm11 +; SSE2-NEXT: pandn %xmm9, %xmm13 +; SSE2-NEXT: pandn %xmm6, %xmm2 +; SSE2-NEXT: por %xmm13, %xmm2 +; SSE2-NEXT: pandn %xmm8, %xmm12 +; SSE2-NEXT: pandn %xmm7, %xmm3 +; SSE2-NEXT: por %xmm12, %xmm3 +; SSE2-NEXT: movdqa %xmm11, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test100: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsb %xmm4, %xmm0 +; SSE4-NEXT: pmaxsb %xmm5, %xmm1 +; SSE4-NEXT: pmaxsb %xmm6, %xmm2 +; SSE4-NEXT: pmaxsb %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test100: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxsb %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxsb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxsb %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxsb %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test100: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsb %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxsb %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test100: -; AVX512BW: vpmaxsb {{.*}} -} - -define void @test101(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <64 x i8>* - %ptr.b = bitcast i8* %gep.b to <64 x i8>* - %load.a = load <64 x i8>, <64 x i8>* %ptr.a, align 2 - %load.b = load <64 x i8>, <64 x i8>* %ptr.b, align 2 - %cmp = icmp ult <64 x i8> %load.a, %load.b - %sel = select <64 x i1> %cmp, <64 x i8> %load.a, <64 x i8> %load.b - store <64 x i8> %sel, <64 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxsb %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sge <64 x i8> %a, %b + %sel = select <64 x i1> %cmp, <64 x i8> %a, <64 x i8> %b + ret <64 x i8> %sel +} + +define <64 x i8> @test101(<64 x i8> %a, <64 x i8> %b) { +; SSE-LABEL: test101: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminub %xmm4, %xmm0 +; SSE-NEXT: pminub %xmm5, %xmm1 +; SSE-NEXT: pminub %xmm6, %xmm2 +; SSE-NEXT: pminub %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX1-LABEL: test101: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminub %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminub %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminub %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminub %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test101: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminub %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminub %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test101: -; AVX512BW: vpminub {{.*}} -} - -define void @test102(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <64 x i8>* - %ptr.b = bitcast i8* %gep.b to <64 x i8>* - %load.a = load <64 x i8>, <64 x i8>* %ptr.a, align 2 - %load.b = load <64 x i8>, <64 x i8>* %ptr.b, align 2 - %cmp = icmp ule <64 x i8> %load.a, %load.b - %sel = select <64 x i1> %cmp, <64 x i8> %load.a, <64 x i8> %load.b - store <64 x i8> %sel, <64 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminub %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ult <64 x i8> %a, %b + %sel = select <64 x i1> %cmp, <64 x i8> %a, <64 x i8> %b + ret <64 x i8> %sel +} + +define <64 x i8> @test102(<64 x i8> %a, <64 x i8> %b) { +; SSE-LABEL: test102: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminub %xmm4, %xmm0 +; SSE-NEXT: pminub %xmm5, %xmm1 +; SSE-NEXT: pminub %xmm6, %xmm2 +; SSE-NEXT: pminub %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX1-LABEL: test102: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminub %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminub %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminub %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminub %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test102: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminub %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminub %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test102: -; AVX512BW: vpminub {{.*}} -} - -define void @test103(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <64 x i8>* - %ptr.b = bitcast i8* %gep.b to <64 x i8>* - %load.a = load <64 x i8>, <64 x i8>* %ptr.a, align 2 - %load.b = load <64 x i8>, <64 x i8>* %ptr.b, align 2 - %cmp = icmp ugt <64 x i8> %load.a, %load.b - %sel = select <64 x i1> %cmp, <64 x i8> %load.a, <64 x i8> %load.b - store <64 x i8> %sel, <64 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminub %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ule <64 x i8> %a, %b + %sel = select <64 x i1> %cmp, <64 x i8> %a, <64 x i8> %b + ret <64 x i8> %sel +} + +define <64 x i8> @test103(<64 x i8> %a, <64 x i8> %b) { +; SSE-LABEL: test103: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxub %xmm4, %xmm0 +; SSE-NEXT: pmaxub %xmm5, %xmm1 +; SSE-NEXT: pmaxub %xmm6, %xmm2 +; SSE-NEXT: pmaxub %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX1-LABEL: test103: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxub %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxub %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxub %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxub %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test103: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxub %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxub %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test103: -; AVX512BW: vpmaxub {{.*}} -} - -define void @test104(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <64 x i8>* - %ptr.b = bitcast i8* %gep.b to <64 x i8>* - %load.a = load <64 x i8>, <64 x i8>* %ptr.a, align 2 - %load.b = load <64 x i8>, <64 x i8>* %ptr.b, align 2 - %cmp = icmp uge <64 x i8> %load.a, %load.b - %sel = select <64 x i1> %cmp, <64 x i8> %load.a, <64 x i8> %load.b - store <64 x i8> %sel, <64 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxub %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ugt <64 x i8> %a, %b + %sel = select <64 x i1> %cmp, <64 x i8> %a, <64 x i8> %b + ret <64 x i8> %sel +} + +define <64 x i8> @test104(<64 x i8> %a, <64 x i8> %b) { +; SSE-LABEL: test104: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxub %xmm4, %xmm0 +; SSE-NEXT: pmaxub %xmm5, %xmm1 +; SSE-NEXT: pmaxub %xmm6, %xmm2 +; SSE-NEXT: pmaxub %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX1-LABEL: test104: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxub %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxub %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxub %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxub %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test104: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxub %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxub %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test104: -; AVX512BW: vpmaxub {{.*}} -} - -define void @test105(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <32 x i16>* - %ptr.b = bitcast i16* %gep.b to <32 x i16>* - %load.a = load <32 x i16>, <32 x i16>* %ptr.a, align 2 - %load.b = load <32 x i16>, <32 x i16>* %ptr.b, align 2 - %cmp = icmp slt <32 x i16> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i16> %load.a, <32 x i16> %load.b - store <32 x i16> %sel, <32 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxub %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp uge <64 x i8> %a, %b + %sel = select <64 x i1> %cmp, <64 x i8> %a, <64 x i8> %b + ret <64 x i8> %sel +} + +define <32 x i16> @test105(<32 x i16> %a, <32 x i16> %b) { +; SSE-LABEL: test105: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminsw %xmm4, %xmm0 +; SSE-NEXT: pminsw %xmm5, %xmm1 +; SSE-NEXT: pminsw %xmm6, %xmm2 +; SSE-NEXT: pminsw %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX1-LABEL: test105: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminsw %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminsw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminsw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminsw %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test105: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminsw %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test105: -; AVX512BW: vpminsw {{.*}} -} - -define void @test106(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <32 x i16>* - %ptr.b = bitcast i16* %gep.b to <32 x i16>* - %load.a = load <32 x i16>, <32 x i16>* %ptr.a, align 2 - %load.b = load <32 x i16>, <32 x i16>* %ptr.b, align 2 - %cmp = icmp sle <32 x i16> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i16> %load.a, <32 x i16> %load.b - store <32 x i16> %sel, <32 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminsw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp slt <32 x i16> %a, %b + %sel = select <32 x i1> %cmp, <32 x i16> %a, <32 x i16> %b + ret <32 x i16> %sel +} + +define <32 x i16> @test106(<32 x i16> %a, <32 x i16> %b) { +; SSE-LABEL: test106: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminsw %xmm4, %xmm0 +; SSE-NEXT: pminsw %xmm5, %xmm1 +; SSE-NEXT: pminsw %xmm6, %xmm2 +; SSE-NEXT: pminsw %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX1-LABEL: test106: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminsw %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminsw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminsw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminsw %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test106: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminsw %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test106: -; AVX512BW: vpminsw {{.*}} -} - -define void @test107(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <32 x i16>* - %ptr.b = bitcast i16* %gep.b to <32 x i16>* - %load.a = load <32 x i16>, <32 x i16>* %ptr.a, align 2 - %load.b = load <32 x i16>, <32 x i16>* %ptr.b, align 2 - %cmp = icmp sgt <32 x i16> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i16> %load.a, <32 x i16> %load.b - store <32 x i16> %sel, <32 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminsw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sle <32 x i16> %a, %b + %sel = select <32 x i1> %cmp, <32 x i16> %a, <32 x i16> %b + ret <32 x i16> %sel +} + +define <32 x i16> @test107(<32 x i16> %a, <32 x i16> %b) { +; SSE-LABEL: test107: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxsw %xmm4, %xmm0 +; SSE-NEXT: pmaxsw %xmm5, %xmm1 +; SSE-NEXT: pmaxsw %xmm6, %xmm2 +; SSE-NEXT: pmaxsw %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX1-LABEL: test107: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxsw %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxsw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxsw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxsw %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test107: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxsw %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test107: -; AVX512BW: vpmaxsw {{.*}} -} - -define void @test108(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <32 x i16>* - %ptr.b = bitcast i16* %gep.b to <32 x i16>* - %load.a = load <32 x i16>, <32 x i16>* %ptr.a, align 2 - %load.b = load <32 x i16>, <32 x i16>* %ptr.b, align 2 - %cmp = icmp sge <32 x i16> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i16> %load.a, <32 x i16> %load.b - store <32 x i16> %sel, <32 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxsw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sgt <32 x i16> %a, %b + %sel = select <32 x i1> %cmp, <32 x i16> %a, <32 x i16> %b + ret <32 x i16> %sel +} + +define <32 x i16> @test108(<32 x i16> %a, <32 x i16> %b) { +; SSE-LABEL: test108: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxsw %xmm4, %xmm0 +; SSE-NEXT: pmaxsw %xmm5, %xmm1 +; SSE-NEXT: pmaxsw %xmm6, %xmm2 +; SSE-NEXT: pmaxsw %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX1-LABEL: test108: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxsw %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxsw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxsw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxsw %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test108: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxsw %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test108: -; AVX512BW: vpmaxsw {{.*}} -} - -define void @test109(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <32 x i16>* - %ptr.b = bitcast i16* %gep.b to <32 x i16>* - %load.a = load <32 x i16>, <32 x i16>* %ptr.a, align 2 - %load.b = load <32 x i16>, <32 x i16>* %ptr.b, align 2 - %cmp = icmp ult <32 x i16> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i16> %load.a, <32 x i16> %load.b - store <32 x i16> %sel, <32 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxsw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sge <32 x i16> %a, %b + %sel = select <32 x i1> %cmp, <32 x i16> %a, <32 x i16> %b + ret <32 x i16> %sel +} + +define <32 x i16> @test109(<32 x i16> %a, <32 x i16> %b) { +; SSE2-LABEL: test109: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm10 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE2-NEXT: movdqa %xmm3, %xmm9 +; SSE2-NEXT: pxor %xmm10, %xmm9 +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: pxor %xmm10, %xmm8 +; SSE2-NEXT: pcmpgtw %xmm9, %xmm8 +; SSE2-NEXT: movdqa %xmm2, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm6, %xmm9 +; SSE2-NEXT: pxor %xmm10, %xmm9 +; SSE2-NEXT: pcmpgtw %xmm11, %xmm9 +; SSE2-NEXT: movdqa %xmm1, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm5, %xmm12 +; SSE2-NEXT: pxor %xmm10, %xmm12 +; SSE2-NEXT: pcmpgtw %xmm11, %xmm12 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: pxor %xmm4, %xmm10 +; SSE2-NEXT: pcmpgtw %xmm11, %xmm10 +; SSE2-NEXT: pand %xmm10, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm10 +; SSE2-NEXT: por %xmm10, %xmm0 +; SSE2-NEXT: pand %xmm12, %xmm1 +; SSE2-NEXT: pandn %xmm5, %xmm12 +; SSE2-NEXT: por %xmm12, %xmm1 +; SSE2-NEXT: pand %xmm9, %xmm2 +; SSE2-NEXT: pandn %xmm6, %xmm9 +; SSE2-NEXT: por %xmm9, %xmm2 +; SSE2-NEXT: pand %xmm8, %xmm3 +; SSE2-NEXT: pandn %xmm7, %xmm8 +; SSE2-NEXT: por %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test109: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminuw %xmm4, %xmm0 +; SSE4-NEXT: pminuw %xmm5, %xmm1 +; SSE4-NEXT: pminuw %xmm6, %xmm2 +; SSE4-NEXT: pminuw %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test109: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminuw %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminuw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminuw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminuw %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test109: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminuw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminuw %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test109: -; AVX512BW: vpminuw {{.*}} -} - -define void @test110(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <32 x i16>* - %ptr.b = bitcast i16* %gep.b to <32 x i16>* - %load.a = load <32 x i16>, <32 x i16>* %ptr.a, align 2 - %load.b = load <32 x i16>, <32 x i16>* %ptr.b, align 2 - %cmp = icmp ule <32 x i16> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i16> %load.a, <32 x i16> %load.b - store <32 x i16> %sel, <32 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminuw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ult <32 x i16> %a, %b + %sel = select <32 x i1> %cmp, <32 x i16> %a, <32 x i16> %b + ret <32 x i16> %sel +} + +define <32 x i16> @test110(<32 x i16> %a, <32 x i16> %b) { +; SSE2-LABEL: test110: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: movdqa %xmm1, %xmm10 +; SSE2-NEXT: psubusw %xmm7, %xmm3 +; SSE2-NEXT: pxor %xmm12, %xmm12 +; SSE2-NEXT: pcmpeqw %xmm12, %xmm3 +; SSE2-NEXT: psubusw %xmm6, %xmm2 +; SSE2-NEXT: pcmpeqw %xmm12, %xmm2 +; SSE2-NEXT: psubusw %xmm5, %xmm1 +; SSE2-NEXT: pcmpeqw %xmm12, %xmm1 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: psubusw %xmm4, %xmm11 +; SSE2-NEXT: pcmpeqw %xmm12, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm11 +; SSE2-NEXT: por %xmm0, %xmm11 +; SSE2-NEXT: pand %xmm1, %xmm10 +; SSE2-NEXT: pandn %xmm5, %xmm1 +; SSE2-NEXT: por %xmm10, %xmm1 +; SSE2-NEXT: pand %xmm2, %xmm9 +; SSE2-NEXT: pandn %xmm6, %xmm2 +; SSE2-NEXT: por %xmm9, %xmm2 +; SSE2-NEXT: pand %xmm3, %xmm8 +; SSE2-NEXT: pandn %xmm7, %xmm3 +; SSE2-NEXT: por %xmm8, %xmm3 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test110: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminuw %xmm4, %xmm0 +; SSE4-NEXT: pminuw %xmm5, %xmm1 +; SSE4-NEXT: pminuw %xmm6, %xmm2 +; SSE4-NEXT: pminuw %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test110: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminuw %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminuw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminuw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminuw %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test110: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminuw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminuw %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test110: -; AVX512BW: vpminuw {{.*}} -} - -define void @test111(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <32 x i16>* - %ptr.b = bitcast i16* %gep.b to <32 x i16>* - %load.a = load <32 x i16>, <32 x i16>* %ptr.a, align 2 - %load.b = load <32 x i16>, <32 x i16>* %ptr.b, align 2 - %cmp = icmp ugt <32 x i16> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i16> %load.a, <32 x i16> %load.b - store <32 x i16> %sel, <32 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminuw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ule <32 x i16> %a, %b + %sel = select <32 x i1> %cmp, <32 x i16> %a, <32 x i16> %b + ret <32 x i16> %sel +} + +define <32 x i16> @test111(<32 x i16> %a, <32 x i16> %b) { +; SSE2-LABEL: test111: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm11 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE2-NEXT: movdqa %xmm7, %xmm9 +; SSE2-NEXT: pxor %xmm11, %xmm9 +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: pxor %xmm11, %xmm8 +; SSE2-NEXT: pcmpgtw %xmm9, %xmm8 +; SSE2-NEXT: movdqa %xmm6, %xmm10 +; SSE2-NEXT: pxor %xmm11, %xmm10 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: pxor %xmm11, %xmm9 +; SSE2-NEXT: pcmpgtw %xmm10, %xmm9 +; SSE2-NEXT: movdqa %xmm5, %xmm12 +; SSE2-NEXT: pxor %xmm11, %xmm12 +; SSE2-NEXT: movdqa %xmm1, %xmm10 +; SSE2-NEXT: pxor %xmm11, %xmm10 +; SSE2-NEXT: pcmpgtw %xmm12, %xmm10 +; SSE2-NEXT: movdqa %xmm4, %xmm12 +; SSE2-NEXT: pxor %xmm11, %xmm12 +; SSE2-NEXT: pxor %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtw %xmm12, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm11 +; SSE2-NEXT: por %xmm11, %xmm0 +; SSE2-NEXT: pand %xmm10, %xmm1 +; SSE2-NEXT: pandn %xmm5, %xmm10 +; SSE2-NEXT: por %xmm1, %xmm10 +; SSE2-NEXT: pand %xmm9, %xmm2 +; SSE2-NEXT: pandn %xmm6, %xmm9 +; SSE2-NEXT: por %xmm2, %xmm9 +; SSE2-NEXT: pand %xmm8, %xmm3 +; SSE2-NEXT: pandn %xmm7, %xmm8 +; SSE2-NEXT: por %xmm3, %xmm8 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test111: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxuw %xmm4, %xmm0 +; SSE4-NEXT: pmaxuw %xmm5, %xmm1 +; SSE4-NEXT: pmaxuw %xmm6, %xmm2 +; SSE4-NEXT: pmaxuw %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test111: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxuw %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxuw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxuw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxuw %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test111: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxuw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxuw %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test111: -; AVX512BW: vpmaxuw {{.*}} -} - -define void @test112(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <32 x i16>* - %ptr.b = bitcast i16* %gep.b to <32 x i16>* - %load.a = load <32 x i16>, <32 x i16>* %ptr.a, align 2 - %load.b = load <32 x i16>, <32 x i16>* %ptr.b, align 2 - %cmp = icmp uge <32 x i16> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i16> %load.a, <32 x i16> %load.b - store <32 x i16> %sel, <32 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxuw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ugt <32 x i16> %a, %b + %sel = select <32 x i1> %cmp, <32 x i16> %a, <32 x i16> %b + ret <32 x i16> %sel +} + +define <32 x i16> @test112(<32 x i16> %a, <32 x i16> %b) { +; SSE2-LABEL: test112: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: psubusw %xmm3, %xmm8 +; SSE2-NEXT: pxor %xmm9, %xmm9 +; SSE2-NEXT: pcmpeqw %xmm9, %xmm8 +; SSE2-NEXT: movdqa %xmm6, %xmm10 +; SSE2-NEXT: psubusw %xmm2, %xmm10 +; SSE2-NEXT: pcmpeqw %xmm9, %xmm10 +; SSE2-NEXT: movdqa %xmm5, %xmm11 +; SSE2-NEXT: psubusw %xmm1, %xmm11 +; SSE2-NEXT: pcmpeqw %xmm9, %xmm11 +; SSE2-NEXT: movdqa %xmm4, %xmm12 +; SSE2-NEXT: psubusw %xmm0, %xmm12 +; SSE2-NEXT: pcmpeqw %xmm9, %xmm12 +; SSE2-NEXT: pand %xmm12, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm12 +; SSE2-NEXT: por %xmm12, %xmm0 +; SSE2-NEXT: pand %xmm11, %xmm1 +; SSE2-NEXT: pandn %xmm5, %xmm11 +; SSE2-NEXT: por %xmm11, %xmm1 +; SSE2-NEXT: pand %xmm10, %xmm2 +; SSE2-NEXT: pandn %xmm6, %xmm10 +; SSE2-NEXT: por %xmm10, %xmm2 +; SSE2-NEXT: pand %xmm8, %xmm3 +; SSE2-NEXT: pandn %xmm7, %xmm8 +; SSE2-NEXT: por %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test112: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxuw %xmm4, %xmm0 +; SSE4-NEXT: pmaxuw %xmm5, %xmm1 +; SSE4-NEXT: pmaxuw %xmm6, %xmm2 +; SSE4-NEXT: pmaxuw %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test112: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxuw %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxuw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxuw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxuw %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test112: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxuw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxuw %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test112: -; AVX512BW: vpmaxuw {{.*}} -} - -define void @test113(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <16 x i32>* - %ptr.b = bitcast i32* %gep.b to <16 x i32>* - %load.a = load <16 x i32>, <16 x i32>* %ptr.a, align 2 - %load.b = load <16 x i32>, <16 x i32>* %ptr.b, align 2 - %cmp = icmp slt <16 x i32> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i32> %load.a, <16 x i32> %load.b - store <16 x i32> %sel, <16 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxuw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp uge <32 x i16> %a, %b + %sel = select <32 x i1> %cmp, <32 x i16> %a, <32 x i16> %b + ret <32 x i16> %sel +} + +define <16 x i32> @test113(<16 x i32> %a, <16 x i32> %b) { +; SSE2-LABEL: test113: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm8 +; SSE2-NEXT: movdqa %xmm6, %xmm9 +; SSE2-NEXT: pcmpgtd %xmm2, %xmm9 +; SSE2-NEXT: movdqa %xmm5, %xmm10 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm10 +; SSE2-NEXT: movdqa %xmm4, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm11 +; SSE2-NEXT: por %xmm11, %xmm0 +; SSE2-NEXT: pand %xmm10, %xmm1 +; SSE2-NEXT: pandn %xmm5, %xmm10 +; SSE2-NEXT: por %xmm10, %xmm1 +; SSE2-NEXT: pand %xmm9, %xmm2 +; SSE2-NEXT: pandn %xmm6, %xmm9 +; SSE2-NEXT: por %xmm9, %xmm2 +; SSE2-NEXT: pand %xmm8, %xmm3 +; SSE2-NEXT: pandn %xmm7, %xmm8 +; SSE2-NEXT: por %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test113: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsd %xmm4, %xmm0 +; SSE4-NEXT: pminsd %xmm5, %xmm1 +; SSE4-NEXT: pminsd %xmm6, %xmm2 +; SSE4-NEXT: pminsd %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test113: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminsd %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminsd %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminsd %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminsd %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test113: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsd %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminsd %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test113: -; AVX512F: vpminsd {{.*}} -} - -define void @test114(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <16 x i32>* - %ptr.b = bitcast i32* %gep.b to <16 x i32>* - %load.a = load <16 x i32>, <16 x i32>* %ptr.a, align 2 - %load.b = load <16 x i32>, <16 x i32>* %ptr.b, align 2 - %cmp = icmp sle <16 x i32> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i32> %load.a, <16 x i32> %load.b - store <16 x i32> %sel, <16 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsd %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp slt <16 x i32> %a, %b + %sel = select <16 x i1> %cmp, <16 x i32> %a, <16 x i32> %b + ret <16 x i32> %sel +} + +define <16 x i32> @test114(<16 x i32> %a, <16 x i32> %b) { +; SSE2-LABEL: test114: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: movdqa %xmm8, %xmm12 +; SSE2-NEXT: pcmpgtd %xmm7, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm13, %xmm13 +; SSE2-NEXT: movdqa %xmm12, %xmm3 +; SSE2-NEXT: pxor %xmm13, %xmm3 +; SSE2-NEXT: movdqa %xmm9, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm14 +; SSE2-NEXT: movdqa %xmm14, %xmm2 +; SSE2-NEXT: pxor %xmm13, %xmm2 +; SSE2-NEXT: movdqa %xmm1, %xmm15 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm15 +; SSE2-NEXT: movdqa %xmm15, %xmm10 +; SSE2-NEXT: pxor %xmm13, %xmm10 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm11 +; SSE2-NEXT: pxor %xmm11, %xmm13 +; SSE2-NEXT: pandn %xmm0, %xmm11 +; SSE2-NEXT: pandn %xmm4, %xmm13 +; SSE2-NEXT: por %xmm13, %xmm11 +; SSE2-NEXT: pandn %xmm1, %xmm15 +; SSE2-NEXT: pandn %xmm5, %xmm10 +; SSE2-NEXT: por %xmm15, %xmm10 +; SSE2-NEXT: pandn %xmm9, %xmm14 +; SSE2-NEXT: pandn %xmm6, %xmm2 +; SSE2-NEXT: por %xmm14, %xmm2 +; SSE2-NEXT: pandn %xmm8, %xmm12 +; SSE2-NEXT: pandn %xmm7, %xmm3 +; SSE2-NEXT: por %xmm12, %xmm3 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test114: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsd %xmm4, %xmm0 +; SSE4-NEXT: pminsd %xmm5, %xmm1 +; SSE4-NEXT: pminsd %xmm6, %xmm2 +; SSE4-NEXT: pminsd %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test114: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminsd %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminsd %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminsd %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminsd %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test114: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsd %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminsd %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test114: -; AVX512F: vpminsd {{.*}} -} - -define void @test115(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <16 x i32>* - %ptr.b = bitcast i32* %gep.b to <16 x i32>* - %load.a = load <16 x i32>, <16 x i32>* %ptr.a, align 2 - %load.b = load <16 x i32>, <16 x i32>* %ptr.b, align 2 - %cmp = icmp sgt <16 x i32> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i32> %load.a, <16 x i32> %load.b - store <16 x i32> %sel, <16 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsd %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sle <16 x i32> %a, %b + %sel = select <16 x i1> %cmp, <16 x i32> %a, <16 x i32> %b + ret <16 x i32> %sel +} + +define <16 x i32> @test115(<16 x i32> %a, <16 x i32> %b) { +; SSE2-LABEL: test115: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: pcmpgtd %xmm7, %xmm3 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm9 +; SSE2-NEXT: movdqa %xmm1, %xmm10 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm10 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm11 +; SSE2-NEXT: por %xmm0, %xmm11 +; SSE2-NEXT: pand %xmm10, %xmm1 +; SSE2-NEXT: pandn %xmm5, %xmm10 +; SSE2-NEXT: por %xmm1, %xmm10 +; SSE2-NEXT: pand %xmm9, %xmm2 +; SSE2-NEXT: pandn %xmm6, %xmm9 +; SSE2-NEXT: por %xmm2, %xmm9 +; SSE2-NEXT: pand %xmm3, %xmm8 +; SSE2-NEXT: pandn %xmm7, %xmm3 +; SSE2-NEXT: por %xmm8, %xmm3 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test115: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsd %xmm4, %xmm0 +; SSE4-NEXT: pmaxsd %xmm5, %xmm1 +; SSE4-NEXT: pmaxsd %xmm6, %xmm2 +; SSE4-NEXT: pmaxsd %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test115: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxsd %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxsd %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxsd %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxsd %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test115: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsd %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxsd %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test115: -; AVX512F: vpmaxsd {{.*}} -} - -define void @test116(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <16 x i32>* - %ptr.b = bitcast i32* %gep.b to <16 x i32>* - %load.a = load <16 x i32>, <16 x i32>* %ptr.a, align 2 - %load.b = load <16 x i32>, <16 x i32>* %ptr.b, align 2 - %cmp = icmp sge <16 x i32> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i32> %load.a, <16 x i32> %load.b - store <16 x i32> %sel, <16 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsd %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sgt <16 x i32> %a, %b + %sel = select <16 x i1> %cmp, <16 x i32> %a, <16 x i32> %b + ret <16 x i32> %sel +} + +define <16 x i32> @test116(<16 x i32> %a, <16 x i32> %b) { +; SSE2-LABEL: test116: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: movdqa %xmm0, %xmm10 +; SSE2-NEXT: movdqa %xmm7, %xmm12 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE2-NEXT: movdqa %xmm12, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm3 +; SSE2-NEXT: movdqa %xmm6, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm9, %xmm13 +; SSE2-NEXT: movdqa %xmm13, %xmm2 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm5, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm14 +; SSE2-NEXT: movdqa %xmm14, %xmm11 +; SSE2-NEXT: pxor %xmm0, %xmm11 +; SSE2-NEXT: movdqa %xmm4, %xmm15 +; SSE2-NEXT: pcmpgtd %xmm10, %xmm15 +; SSE2-NEXT: pxor %xmm15, %xmm0 +; SSE2-NEXT: pandn %xmm10, %xmm15 +; SSE2-NEXT: pandn %xmm4, %xmm0 +; SSE2-NEXT: por %xmm15, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm14 +; SSE2-NEXT: pandn %xmm5, %xmm11 +; SSE2-NEXT: por %xmm14, %xmm11 +; SSE2-NEXT: pandn %xmm9, %xmm13 +; SSE2-NEXT: pandn %xmm6, %xmm2 +; SSE2-NEXT: por %xmm13, %xmm2 +; SSE2-NEXT: pandn %xmm8, %xmm12 +; SSE2-NEXT: pandn %xmm7, %xmm3 +; SSE2-NEXT: por %xmm12, %xmm3 +; SSE2-NEXT: movdqa %xmm11, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test116: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsd %xmm4, %xmm0 +; SSE4-NEXT: pmaxsd %xmm5, %xmm1 +; SSE4-NEXT: pmaxsd %xmm6, %xmm2 +; SSE4-NEXT: pmaxsd %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test116: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxsd %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxsd %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxsd %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxsd %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test116: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsd %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxsd %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test116: -; AVX512F: vpmaxsd {{.*}} -} - -define void @test117(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <16 x i32>* - %ptr.b = bitcast i32* %gep.b to <16 x i32>* - %load.a = load <16 x i32>, <16 x i32>* %ptr.a, align 2 - %load.b = load <16 x i32>, <16 x i32>* %ptr.b, align 2 - %cmp = icmp ult <16 x i32> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i32> %load.a, <16 x i32> %load.b - store <16 x i32> %sel, <16 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsd %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sge <16 x i32> %a, %b + %sel = select <16 x i1> %cmp, <16 x i32> %a, <16 x i32> %b + ret <16 x i32> %sel +} + +define <16 x i32> @test117(<16 x i32> %a, <16 x i32> %b) { +; SSE2-LABEL: test117: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm10 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm9 +; SSE2-NEXT: pxor %xmm10, %xmm9 +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: pxor %xmm10, %xmm8 +; SSE2-NEXT: pcmpgtd %xmm9, %xmm8 +; SSE2-NEXT: movdqa %xmm2, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm6, %xmm9 +; SSE2-NEXT: pxor %xmm10, %xmm9 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm9 +; SSE2-NEXT: movdqa %xmm1, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm5, %xmm12 +; SSE2-NEXT: pxor %xmm10, %xmm12 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm12 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: pxor %xmm4, %xmm10 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm10 +; SSE2-NEXT: pand %xmm10, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm10 +; SSE2-NEXT: por %xmm10, %xmm0 +; SSE2-NEXT: pand %xmm12, %xmm1 +; SSE2-NEXT: pandn %xmm5, %xmm12 +; SSE2-NEXT: por %xmm12, %xmm1 +; SSE2-NEXT: pand %xmm9, %xmm2 +; SSE2-NEXT: pandn %xmm6, %xmm9 +; SSE2-NEXT: por %xmm9, %xmm2 +; SSE2-NEXT: pand %xmm8, %xmm3 +; SSE2-NEXT: pandn %xmm7, %xmm8 +; SSE2-NEXT: por %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test117: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminud %xmm4, %xmm0 +; SSE4-NEXT: pminud %xmm5, %xmm1 +; SSE4-NEXT: pminud %xmm6, %xmm2 +; SSE4-NEXT: pminud %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test117: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminud %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminud %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminud %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminud %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test117: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminud %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminud %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test117: -; AVX512F: vpminud {{.*}} -} - -define void @test118(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <16 x i32>* - %ptr.b = bitcast i32* %gep.b to <16 x i32>* - %load.a = load <16 x i32>, <16 x i32>* %ptr.a, align 2 - %load.b = load <16 x i32>, <16 x i32>* %ptr.b, align 2 - %cmp = icmp ule <16 x i32> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i32> %load.a, <16 x i32> %load.b - store <16 x i32> %sel, <16 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminud %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ult <16 x i32> %a, %b + %sel = select <16 x i1> %cmp, <16 x i32> %a, <16 x i32> %b + ret <16 x i32> %sel +} + +define <16 x i32> @test118(<16 x i32> %a, <16 x i32> %b) { +; SSE2-LABEL: test118: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm10 +; SSE2-NEXT: movdqa {{.*#+}} xmm14 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm7, %xmm0 +; SSE2-NEXT: pxor %xmm14, %xmm0 +; SSE2-NEXT: movdqa %xmm3, %xmm12 +; SSE2-NEXT: pxor %xmm14, %xmm12 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE2-NEXT: movdqa %xmm12, %xmm8 +; SSE2-NEXT: pxor %xmm0, %xmm8 +; SSE2-NEXT: movdqa %xmm6, %xmm9 +; SSE2-NEXT: pxor %xmm14, %xmm9 +; SSE2-NEXT: movdqa %xmm2, %xmm13 +; SSE2-NEXT: pxor %xmm14, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm9, %xmm13 +; SSE2-NEXT: movdqa %xmm13, %xmm9 +; SSE2-NEXT: pxor %xmm0, %xmm9 +; SSE2-NEXT: movdqa %xmm5, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm11 +; SSE2-NEXT: movdqa %xmm1, %xmm15 +; SSE2-NEXT: pxor %xmm14, %xmm15 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm15 +; SSE2-NEXT: movdqa %xmm4, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: movdqa %xmm15, %xmm11 +; SSE2-NEXT: pxor %xmm0, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm0 +; SSE2-NEXT: pandn %xmm10, %xmm14 +; SSE2-NEXT: pandn %xmm4, %xmm0 +; SSE2-NEXT: por %xmm14, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm15 +; SSE2-NEXT: pandn %xmm5, %xmm11 +; SSE2-NEXT: por %xmm15, %xmm11 +; SSE2-NEXT: pandn %xmm2, %xmm13 +; SSE2-NEXT: pandn %xmm6, %xmm9 +; SSE2-NEXT: por %xmm13, %xmm9 +; SSE2-NEXT: pandn %xmm3, %xmm12 +; SSE2-NEXT: pandn %xmm7, %xmm8 +; SSE2-NEXT: por %xmm12, %xmm8 +; SSE2-NEXT: movdqa %xmm11, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test118: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminud %xmm4, %xmm0 +; SSE4-NEXT: pminud %xmm5, %xmm1 +; SSE4-NEXT: pminud %xmm6, %xmm2 +; SSE4-NEXT: pminud %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test118: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminud %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminud %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminud %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminud %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test118: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminud %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminud %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test118: -; AVX512F: vpminud {{.*}} -} - -define void @test119(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <16 x i32>* - %ptr.b = bitcast i32* %gep.b to <16 x i32>* - %load.a = load <16 x i32>, <16 x i32>* %ptr.a, align 2 - %load.b = load <16 x i32>, <16 x i32>* %ptr.b, align 2 - %cmp = icmp ugt <16 x i32> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i32> %load.a, <16 x i32> %load.b - store <16 x i32> %sel, <16 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminud %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ule <16 x i32> %a, %b + %sel = select <16 x i1> %cmp, <16 x i32> %a, <16 x i32> %b + ret <16 x i32> %sel +} + +define <16 x i32> @test119(<16 x i32> %a, <16 x i32> %b) { +; SSE2-LABEL: test119: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm11 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm7, %xmm9 +; SSE2-NEXT: pxor %xmm11, %xmm9 +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: pxor %xmm11, %xmm8 +; SSE2-NEXT: pcmpgtd %xmm9, %xmm8 +; SSE2-NEXT: movdqa %xmm6, %xmm10 +; SSE2-NEXT: pxor %xmm11, %xmm10 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: pxor %xmm11, %xmm9 +; SSE2-NEXT: pcmpgtd %xmm10, %xmm9 +; SSE2-NEXT: movdqa %xmm5, %xmm12 +; SSE2-NEXT: pxor %xmm11, %xmm12 +; SSE2-NEXT: movdqa %xmm1, %xmm10 +; SSE2-NEXT: pxor %xmm11, %xmm10 +; SSE2-NEXT: pcmpgtd %xmm12, %xmm10 +; SSE2-NEXT: movdqa %xmm4, %xmm12 +; SSE2-NEXT: pxor %xmm11, %xmm12 +; SSE2-NEXT: pxor %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm12, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm11 +; SSE2-NEXT: por %xmm11, %xmm0 +; SSE2-NEXT: pand %xmm10, %xmm1 +; SSE2-NEXT: pandn %xmm5, %xmm10 +; SSE2-NEXT: por %xmm1, %xmm10 +; SSE2-NEXT: pand %xmm9, %xmm2 +; SSE2-NEXT: pandn %xmm6, %xmm9 +; SSE2-NEXT: por %xmm2, %xmm9 +; SSE2-NEXT: pand %xmm8, %xmm3 +; SSE2-NEXT: pandn %xmm7, %xmm8 +; SSE2-NEXT: por %xmm3, %xmm8 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test119: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxud %xmm4, %xmm0 +; SSE4-NEXT: pmaxud %xmm5, %xmm1 +; SSE4-NEXT: pmaxud %xmm6, %xmm2 +; SSE4-NEXT: pmaxud %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test119: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxud %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxud %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxud %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxud %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test119: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxud %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxud %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test119: -; AVX512F: vpmaxud {{.*}} -} - -define void @test120(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <16 x i32>* - %ptr.b = bitcast i32* %gep.b to <16 x i32>* - %load.a = load <16 x i32>, <16 x i32>* %ptr.a, align 2 - %load.b = load <16 x i32>, <16 x i32>* %ptr.b, align 2 - %cmp = icmp uge <16 x i32> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i32> %load.a, <16 x i32> %load.b - store <16 x i32> %sel, <16 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxud %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ugt <16 x i32> %a, %b + %sel = select <16 x i1> %cmp, <16 x i32> %a, <16 x i32> %b + ret <16 x i32> %sel +} + +define <16 x i32> @test120(<16 x i32> %a, <16 x i32> %b) { +; SSE2-LABEL: test120: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm10 +; SSE2-NEXT: movdqa {{.*#+}} xmm14 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm0 +; SSE2-NEXT: pxor %xmm14, %xmm0 +; SSE2-NEXT: movdqa %xmm7, %xmm12 +; SSE2-NEXT: pxor %xmm14, %xmm12 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE2-NEXT: movdqa %xmm12, %xmm8 +; SSE2-NEXT: pxor %xmm0, %xmm8 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: pxor %xmm14, %xmm9 +; SSE2-NEXT: movdqa %xmm6, %xmm13 +; SSE2-NEXT: pxor %xmm14, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm9, %xmm13 +; SSE2-NEXT: movdqa %xmm13, %xmm9 +; SSE2-NEXT: pxor %xmm0, %xmm9 +; SSE2-NEXT: movdqa %xmm1, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm11 +; SSE2-NEXT: movdqa %xmm5, %xmm15 +; SSE2-NEXT: pxor %xmm14, %xmm15 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm15 +; SSE2-NEXT: movdqa %xmm10, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm11 +; SSE2-NEXT: pxor %xmm4, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: movdqa %xmm15, %xmm11 +; SSE2-NEXT: pxor %xmm0, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm0 +; SSE2-NEXT: pandn %xmm10, %xmm14 +; SSE2-NEXT: pandn %xmm4, %xmm0 +; SSE2-NEXT: por %xmm14, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm15 +; SSE2-NEXT: pandn %xmm5, %xmm11 +; SSE2-NEXT: por %xmm15, %xmm11 +; SSE2-NEXT: pandn %xmm2, %xmm13 +; SSE2-NEXT: pandn %xmm6, %xmm9 +; SSE2-NEXT: por %xmm13, %xmm9 +; SSE2-NEXT: pandn %xmm3, %xmm12 +; SSE2-NEXT: pandn %xmm7, %xmm8 +; SSE2-NEXT: por %xmm12, %xmm8 +; SSE2-NEXT: movdqa %xmm11, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test120: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxud %xmm4, %xmm0 +; SSE4-NEXT: pmaxud %xmm5, %xmm1 +; SSE4-NEXT: pmaxud %xmm6, %xmm2 +; SSE4-NEXT: pmaxud %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test120: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxud %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxud %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxud %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxud %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test120: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxud %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxud %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test120: -; AVX512F: vpmaxud {{.*}} -} - -define void @test121(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i64>* - %ptr.b = bitcast i32* %gep.b to <8 x i64>* - %load.a = load <8 x i64>, <8 x i64>* %ptr.a, align 2 - %load.b = load <8 x i64>, <8 x i64>* %ptr.b, align 2 - %cmp = icmp slt <8 x i64> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i64> %load.a, <8 x i64> %load.b - store <8 x i64> %sel, <8 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxud %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp uge <16 x i32> %a, %b + %sel = select <16 x i1> %cmp, <16 x i32> %a, <16 x i32> %b + ret <16 x i32> %sel +} + +define <8 x i64> @test121(<8 x i64> %a, <8 x i64> %b) { +; SSE2-LABEL: test121: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: pxor %xmm9, %xmm8 +; SSE2-NEXT: movdqa %xmm7, %xmm10 +; SSE2-NEXT: pxor %xmm9, %xmm10 +; SSE2-NEXT: movdqa %xmm10, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm8, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3] +; SSE2-NEXT: pand %xmm12, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm11[1,1,3,3] +; SSE2-NEXT: por %xmm10, %xmm8 +; SSE2-NEXT: movdqa %xmm2, %xmm10 +; SSE2-NEXT: pxor %xmm9, %xmm10 +; SSE2-NEXT: movdqa %xmm6, %xmm11 +; SSE2-NEXT: pxor %xmm9, %xmm11 +; SSE2-NEXT: movdqa %xmm11, %xmm12 +; SSE2-NEXT: pcmpgtd %xmm10, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm10, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm11[1,1,3,3] +; SSE2-NEXT: pand %xmm13, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm12[1,1,3,3] +; SSE2-NEXT: por %xmm11, %xmm10 +; SSE2-NEXT: movdqa %xmm1, %xmm11 +; SSE2-NEXT: pxor %xmm9, %xmm11 +; SSE2-NEXT: movdqa %xmm5, %xmm12 +; SSE2-NEXT: pxor %xmm9, %xmm12 +; SSE2-NEXT: movdqa %xmm12, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm11, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm12[1,1,3,3] +; SSE2-NEXT: pand %xmm14, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm13[1,1,3,3] +; SSE2-NEXT: por %xmm11, %xmm12 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pxor %xmm9, %xmm11 +; SSE2-NEXT: pxor %xmm4, %xmm9 +; SSE2-NEXT: movdqa %xmm9, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm11, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3] +; SSE2-NEXT: pand %xmm14, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[1,1,3,3] +; SSE2-NEXT: por %xmm9, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm11 +; SSE2-NEXT: por %xmm11, %xmm0 +; SSE2-NEXT: pand %xmm12, %xmm1 +; SSE2-NEXT: pandn %xmm5, %xmm12 +; SSE2-NEXT: por %xmm12, %xmm1 +; SSE2-NEXT: pand %xmm10, %xmm2 +; SSE2-NEXT: pandn %xmm6, %xmm10 +; SSE2-NEXT: por %xmm10, %xmm2 +; SSE2-NEXT: pand %xmm8, %xmm3 +; SSE2-NEXT: pandn %xmm7, %xmm8 +; SSE2-NEXT: por %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test121: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm8 +; SSE4-NEXT: movdqa %xmm7, %xmm9 +; SSE4-NEXT: pcmpgtq %xmm3, %xmm9 +; SSE4-NEXT: movdqa %xmm6, %xmm10 +; SSE4-NEXT: pcmpgtq %xmm2, %xmm10 +; SSE4-NEXT: movdqa %xmm5, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm1, %xmm11 +; SSE4-NEXT: movdqa %xmm4, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm8, %xmm0 +; SSE4-NEXT: blendvpd %xmm8, %xmm4 +; SSE4-NEXT: movdqa %xmm11, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm5 +; SSE4-NEXT: movdqa %xmm10, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm6 +; SSE4-NEXT: movdqa %xmm9, %xmm0 +; SSE4-NEXT: blendvpd %xmm3, %xmm7 +; SSE4-NEXT: movapd %xmm4, %xmm0 +; SSE4-NEXT: movapd %xmm5, %xmm1 +; SSE4-NEXT: movapd %xmm6, %xmm2 +; SSE4-NEXT: movapd %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test121: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm1, %xmm3, %xmm5 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm5, %xmm6, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm0, %xmm2, %xmm6 +; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm6, %ymm5 +; AVX1-NEXT: vblendvpd %ymm5, %ymm0, %ymm2, %ymm0 +; AVX1-NEXT: vblendvpd %ymm4, %ymm1, %ymm3, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test121: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %ymm1, %ymm3, %ymm4 +; AVX2-NEXT: vpcmpgtq %ymm0, %ymm2, %ymm5 +; AVX2-NEXT: vblendvpd %ymm5, %ymm0, %ymm2, %ymm0 +; AVX2-NEXT: vblendvpd %ymm4, %ymm1, %ymm3, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test121: -; AVX512F: vpminsq {{.*}} -} - -define void @test122(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i64>* - %ptr.b = bitcast i32* %gep.b to <8 x i64>* - %load.a = load <8 x i64>, <8 x i64>* %ptr.a, align 2 - %load.b = load <8 x i64>, <8 x i64>* %ptr.b, align 2 - %cmp = icmp sle <8 x i64> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i64> %load.a, <8 x i64> %load.b - store <8 x i64> %sel, <8 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp slt <8 x i64> %a, %b + %sel = select <8 x i1> %cmp, <8 x i64> %a, <8 x i64> %b + ret <8 x i64> %sel +} + +define <8 x i64> @test122(<8 x i64> %a, <8 x i64> %b) { +; SSE2-LABEL: test122: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: movdqa %xmm8, -{{[0-9]+}}(%rsp) # 16-byte Spill +; SSE2-NEXT: movdqa %xmm3, %xmm7 +; SSE2-NEXT: movdqa %xmm2, %xmm3 +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm0, %xmm9 +; SSE2-NEXT: movdqa {{.*#+}} xmm10 = [2147483648,0,2147483648,0] +; SSE2-NEXT: pxor %xmm10, %xmm8 +; SSE2-NEXT: movdqa %xmm7, %xmm0 +; SSE2-NEXT: pxor %xmm10, %xmm0 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm8, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE2-NEXT: pand %xmm12, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: movdqa %xmm12, %xmm8 +; SSE2-NEXT: pxor %xmm1, %xmm8 +; SSE2-NEXT: movdqa %xmm6, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm3, %xmm13 +; SSE2-NEXT: pxor %xmm10, %xmm13 +; SSE2-NEXT: movdqa %xmm13, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm11, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[1,1,3,3] +; SSE2-NEXT: pand %xmm15, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm14[1,1,3,3] +; SSE2-NEXT: por %xmm11, %xmm13 +; SSE2-NEXT: movdqa %xmm5, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm2, %xmm14 +; SSE2-NEXT: pxor %xmm10, %xmm14 +; SSE2-NEXT: movdqa %xmm14, %xmm15 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm15 +; SSE2-NEXT: pcmpeqd %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm15[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm14[1,1,3,3] +; SSE2-NEXT: pand %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm15[1,1,3,3] +; SSE2-NEXT: por %xmm14, %xmm15 +; SSE2-NEXT: movdqa %xmm4, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: pxor %xmm9, %xmm10 +; SSE2-NEXT: movdqa %xmm10, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: pcmpeqd %xmm11, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm14[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm13, %xmm10 +; SSE2-NEXT: pxor %xmm1, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm14[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm14 +; SSE2-NEXT: movdqa %xmm15, %xmm11 +; SSE2-NEXT: pxor %xmm1, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm1 +; SSE2-NEXT: pandn %xmm9, %xmm14 +; SSE2-NEXT: pandn %xmm4, %xmm1 +; SSE2-NEXT: por %xmm14, %xmm1 +; SSE2-NEXT: pandn %xmm2, %xmm15 +; SSE2-NEXT: pandn %xmm5, %xmm11 +; SSE2-NEXT: por %xmm15, %xmm11 +; SSE2-NEXT: pandn %xmm3, %xmm13 +; SSE2-NEXT: pandn %xmm6, %xmm10 +; SSE2-NEXT: por %xmm13, %xmm10 +; SSE2-NEXT: pandn %xmm7, %xmm12 +; SSE2-NEXT: pandn -{{[0-9]+}}(%rsp), %xmm8 # 16-byte Folded Reload +; SSE2-NEXT: por %xmm12, %xmm8 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: movdqa %xmm11, %xmm1 +; SSE2-NEXT: movdqa %xmm10, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test122: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm8 +; SSE4-NEXT: movdqa %xmm3, %xmm9 +; SSE4-NEXT: pcmpgtq %xmm7, %xmm9 +; SSE4-NEXT: pcmpeqd %xmm12, %xmm12 +; SSE4-NEXT: pxor %xmm12, %xmm9 +; SSE4-NEXT: movdqa %xmm2, %xmm10 +; SSE4-NEXT: pcmpgtq %xmm6, %xmm10 +; SSE4-NEXT: pxor %xmm12, %xmm10 +; SSE4-NEXT: movdqa %xmm1, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm5, %xmm11 +; SSE4-NEXT: pxor %xmm12, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm4, %xmm0 +; SSE4-NEXT: pxor %xmm12, %xmm0 +; SSE4-NEXT: blendvpd %xmm8, %xmm4 +; SSE4-NEXT: movdqa %xmm11, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm5 +; SSE4-NEXT: movdqa %xmm10, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm6 +; SSE4-NEXT: movdqa %xmm9, %xmm0 +; SSE4-NEXT: blendvpd %xmm3, %xmm7 +; SSE4-NEXT: movapd %xmm4, %xmm0 +; SSE4-NEXT: movapd %xmm5, %xmm1 +; SSE4-NEXT: movapd %xmm6, %xmm2 +; SSE4-NEXT: movapd %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test122: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpcmpeqd %xmm5, %xmm5, %xmm5 +; AVX1-NEXT: vpxor %xmm5, %xmm4, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm1, %xmm6 +; AVX1-NEXT: vpxor %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4 +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm7 +; AVX1-NEXT: vpcmpgtq %xmm6, %xmm7, %xmm6 +; AVX1-NEXT: vpxor %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm0, %xmm7 +; AVX1-NEXT: vpxor %xmm5, %xmm7, %xmm5 +; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5 +; AVX1-NEXT: vblendvpd %ymm5, %ymm0, %ymm2, %ymm0 +; AVX1-NEXT: vblendvpd %ymm4, %ymm1, %ymm3, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test122: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %ymm3, %ymm1, %ymm4 +; AVX2-NEXT: vpcmpeqd %ymm5, %ymm5, %ymm5 +; AVX2-NEXT: vpxor %ymm5, %ymm4, %ymm4 +; AVX2-NEXT: vpcmpgtq %ymm2, %ymm0, %ymm6 +; AVX2-NEXT: vpxor %ymm5, %ymm6, %ymm5 +; AVX2-NEXT: vblendvpd %ymm5, %ymm0, %ymm2, %ymm0 +; AVX2-NEXT: vblendvpd %ymm4, %ymm1, %ymm3, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test122: -; AVX512F: vpminsq {{.*}} -} - -define void @test123(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i64>* - %ptr.b = bitcast i32* %gep.b to <8 x i64>* - %load.a = load <8 x i64>, <8 x i64>* %ptr.a, align 2 - %load.b = load <8 x i64>, <8 x i64>* %ptr.b, align 2 - %cmp = icmp sgt <8 x i64> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i64> %load.a, <8 x i64> %load.b - store <8 x i64> %sel, <8 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sle <8 x i64> %a, %b + %sel = select <8 x i1> %cmp, <8 x i64> %a, <8 x i64> %b + ret <8 x i64> %sel +} + +define <8 x i64> @test123(<8 x i64> %a, <8 x i64> %b) { +; SSE2-LABEL: test123: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: pxor %xmm9, %xmm8 +; SSE2-NEXT: movdqa %xmm3, %xmm10 +; SSE2-NEXT: pxor %xmm9, %xmm10 +; SSE2-NEXT: movdqa %xmm10, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm8, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3] +; SSE2-NEXT: pand %xmm12, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm11[1,1,3,3] +; SSE2-NEXT: por %xmm10, %xmm8 +; SSE2-NEXT: movdqa %xmm6, %xmm10 +; SSE2-NEXT: pxor %xmm9, %xmm10 +; SSE2-NEXT: movdqa %xmm2, %xmm11 +; SSE2-NEXT: pxor %xmm9, %xmm11 +; SSE2-NEXT: movdqa %xmm11, %xmm12 +; SSE2-NEXT: pcmpgtd %xmm10, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm10, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm11[1,1,3,3] +; SSE2-NEXT: pand %xmm13, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm12[1,1,3,3] +; SSE2-NEXT: por %xmm11, %xmm10 +; SSE2-NEXT: movdqa %xmm5, %xmm11 +; SSE2-NEXT: pxor %xmm9, %xmm11 +; SSE2-NEXT: movdqa %xmm1, %xmm12 +; SSE2-NEXT: pxor %xmm9, %xmm12 +; SSE2-NEXT: movdqa %xmm12, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm11, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm12[1,1,3,3] +; SSE2-NEXT: pand %xmm14, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm13[1,1,3,3] +; SSE2-NEXT: por %xmm11, %xmm12 +; SSE2-NEXT: movdqa %xmm4, %xmm11 +; SSE2-NEXT: pxor %xmm9, %xmm11 +; SSE2-NEXT: pxor %xmm0, %xmm9 +; SSE2-NEXT: movdqa %xmm9, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm11, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3] +; SSE2-NEXT: pand %xmm14, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[1,1,3,3] +; SSE2-NEXT: por %xmm9, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm11 +; SSE2-NEXT: por %xmm11, %xmm0 +; SSE2-NEXT: pand %xmm12, %xmm1 +; SSE2-NEXT: pandn %xmm5, %xmm12 +; SSE2-NEXT: por %xmm12, %xmm1 +; SSE2-NEXT: pand %xmm10, %xmm2 +; SSE2-NEXT: pandn %xmm6, %xmm10 +; SSE2-NEXT: por %xmm10, %xmm2 +; SSE2-NEXT: pand %xmm8, %xmm3 +; SSE2-NEXT: pandn %xmm7, %xmm8 +; SSE2-NEXT: por %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test123: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm8 +; SSE4-NEXT: movdqa %xmm3, %xmm9 +; SSE4-NEXT: pcmpgtq %xmm7, %xmm9 +; SSE4-NEXT: movdqa %xmm2, %xmm10 +; SSE4-NEXT: pcmpgtq %xmm6, %xmm10 +; SSE4-NEXT: movdqa %xmm1, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm5, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm4, %xmm0 +; SSE4-NEXT: blendvpd %xmm8, %xmm4 +; SSE4-NEXT: movdqa %xmm11, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm5 +; SSE4-NEXT: movdqa %xmm10, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm6 +; SSE4-NEXT: movdqa %xmm9, %xmm0 +; SSE4-NEXT: blendvpd %xmm3, %xmm7 +; SSE4-NEXT: movapd %xmm4, %xmm0 +; SSE4-NEXT: movapd %xmm5, %xmm1 +; SSE4-NEXT: movapd %xmm6, %xmm2 +; SSE4-NEXT: movapd %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test123: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm1, %xmm5 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm4 +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm5 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm5, %xmm6, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm0, %xmm6 +; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm6, %ymm5 +; AVX1-NEXT: vblendvpd %ymm5, %ymm0, %ymm2, %ymm0 +; AVX1-NEXT: vblendvpd %ymm4, %ymm1, %ymm3, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test123: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %ymm3, %ymm1, %ymm4 +; AVX2-NEXT: vpcmpgtq %ymm2, %ymm0, %ymm5 +; AVX2-NEXT: vblendvpd %ymm5, %ymm0, %ymm2, %ymm0 +; AVX2-NEXT: vblendvpd %ymm4, %ymm1, %ymm3, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test123: -; AVX512F: vpmaxsq {{.*}} -} - -define void @test124(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i64>* - %ptr.b = bitcast i32* %gep.b to <8 x i64>* - %load.a = load <8 x i64>, <8 x i64>* %ptr.a, align 2 - %load.b = load <8 x i64>, <8 x i64>* %ptr.b, align 2 - %cmp = icmp sge <8 x i64> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i64> %load.a, <8 x i64> %load.b - store <8 x i64> %sel, <8 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sgt <8 x i64> %a, %b + %sel = select <8 x i1> %cmp, <8 x i64> %a, <8 x i64> %b + ret <8 x i64> %sel +} + +define <8 x i64> @test124(<8 x i64> %a, <8 x i64> %b) { +; SSE2-LABEL: test124: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm7, %xmm11 +; SSE2-NEXT: movdqa %xmm11, -{{[0-9]+}}(%rsp) # 16-byte Spill +; SSE2-NEXT: movdqa %xmm3, %xmm7 +; SSE2-NEXT: movdqa %xmm2, %xmm3 +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm0, %xmm9 +; SSE2-NEXT: movdqa {{.*#+}} xmm10 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: pxor %xmm10, %xmm8 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: pxor %xmm10, %xmm0 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm8, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE2-NEXT: pand %xmm12, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: movdqa %xmm12, %xmm8 +; SSE2-NEXT: pxor %xmm1, %xmm8 +; SSE2-NEXT: movdqa %xmm3, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm6, %xmm13 +; SSE2-NEXT: pxor %xmm10, %xmm13 +; SSE2-NEXT: movdqa %xmm13, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm11, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[1,1,3,3] +; SSE2-NEXT: pand %xmm15, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm14[1,1,3,3] +; SSE2-NEXT: por %xmm11, %xmm13 +; SSE2-NEXT: movdqa %xmm2, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm5, %xmm14 +; SSE2-NEXT: pxor %xmm10, %xmm14 +; SSE2-NEXT: movdqa %xmm14, %xmm15 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm15 +; SSE2-NEXT: pcmpeqd %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm15[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm14[1,1,3,3] +; SSE2-NEXT: pand %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm15[1,1,3,3] +; SSE2-NEXT: por %xmm14, %xmm15 +; SSE2-NEXT: movdqa %xmm9, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: pxor %xmm4, %xmm10 +; SSE2-NEXT: movdqa %xmm10, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: pcmpeqd %xmm11, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm14[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm13, %xmm10 +; SSE2-NEXT: pxor %xmm1, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm14[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm14 +; SSE2-NEXT: movdqa %xmm15, %xmm11 +; SSE2-NEXT: pxor %xmm1, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm1 +; SSE2-NEXT: pandn %xmm9, %xmm14 +; SSE2-NEXT: pandn %xmm4, %xmm1 +; SSE2-NEXT: por %xmm14, %xmm1 +; SSE2-NEXT: pandn %xmm2, %xmm15 +; SSE2-NEXT: pandn %xmm5, %xmm11 +; SSE2-NEXT: por %xmm15, %xmm11 +; SSE2-NEXT: pandn %xmm3, %xmm13 +; SSE2-NEXT: pandn %xmm6, %xmm10 +; SSE2-NEXT: por %xmm13, %xmm10 +; SSE2-NEXT: pandn %xmm7, %xmm12 +; SSE2-NEXT: pandn -{{[0-9]+}}(%rsp), %xmm8 # 16-byte Folded Reload +; SSE2-NEXT: por %xmm12, %xmm8 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: movdqa %xmm11, %xmm1 +; SSE2-NEXT: movdqa %xmm10, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test124: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm8 +; SSE4-NEXT: movdqa %xmm7, %xmm9 +; SSE4-NEXT: pcmpgtq %xmm3, %xmm9 +; SSE4-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE4-NEXT: pxor %xmm0, %xmm9 +; SSE4-NEXT: movdqa %xmm6, %xmm10 +; SSE4-NEXT: pcmpgtq %xmm2, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: movdqa %xmm5, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm1, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: movdqa %xmm4, %xmm12 +; SSE4-NEXT: pcmpgtq %xmm8, %xmm12 +; SSE4-NEXT: pxor %xmm12, %xmm0 +; SSE4-NEXT: blendvpd %xmm8, %xmm4 +; SSE4-NEXT: movdqa %xmm11, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm5 +; SSE4-NEXT: movdqa %xmm10, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm6 +; SSE4-NEXT: movdqa %xmm9, %xmm0 +; SSE4-NEXT: blendvpd %xmm3, %xmm7 +; SSE4-NEXT: movapd %xmm4, %xmm0 +; SSE4-NEXT: movapd %xmm5, %xmm1 +; SSE4-NEXT: movapd %xmm6, %xmm2 +; SSE4-NEXT: movapd %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test124: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpcmpeqd %xmm5, %xmm5, %xmm5 +; AVX1-NEXT: vpxor %xmm5, %xmm4, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm1, %xmm3, %xmm6 +; AVX1-NEXT: vpxor %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6 +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm7 +; AVX1-NEXT: vpcmpgtq %xmm6, %xmm7, %xmm6 +; AVX1-NEXT: vpxor %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm0, %xmm2, %xmm7 +; AVX1-NEXT: vpxor %xmm5, %xmm7, %xmm5 +; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5 +; AVX1-NEXT: vblendvpd %ymm5, %ymm0, %ymm2, %ymm0 +; AVX1-NEXT: vblendvpd %ymm4, %ymm1, %ymm3, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test124: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %ymm1, %ymm3, %ymm4 +; AVX2-NEXT: vpcmpeqd %ymm5, %ymm5, %ymm5 +; AVX2-NEXT: vpxor %ymm5, %ymm4, %ymm4 +; AVX2-NEXT: vpcmpgtq %ymm0, %ymm2, %ymm6 +; AVX2-NEXT: vpxor %ymm5, %ymm6, %ymm5 +; AVX2-NEXT: vblendvpd %ymm5, %ymm0, %ymm2, %ymm0 +; AVX2-NEXT: vblendvpd %ymm4, %ymm1, %ymm3, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test124: -; AVX512F: vpmaxsq {{.*}} -} - -define void @test125(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i64>* - %ptr.b = bitcast i32* %gep.b to <8 x i64>* - %load.a = load <8 x i64>, <8 x i64>* %ptr.a, align 2 - %load.b = load <8 x i64>, <8 x i64>* %ptr.b, align 2 - %cmp = icmp ult <8 x i64> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i64> %load.a, <8 x i64> %load.b - store <8 x i64> %sel, <8 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sge <8 x i64> %a, %b + %sel = select <8 x i1> %cmp, <8 x i64> %a, <8 x i64> %b + ret <8 x i64> %sel +} + +define <8 x i64> @test125(<8 x i64> %a, <8 x i64> %b) { +; SSE2-LABEL: test125: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: pxor %xmm9, %xmm8 +; SSE2-NEXT: movdqa %xmm7, %xmm10 +; SSE2-NEXT: pxor %xmm9, %xmm10 +; SSE2-NEXT: movdqa %xmm10, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm8, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3] +; SSE2-NEXT: pand %xmm12, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm11[1,1,3,3] +; SSE2-NEXT: por %xmm10, %xmm8 +; SSE2-NEXT: movdqa %xmm2, %xmm10 +; SSE2-NEXT: pxor %xmm9, %xmm10 +; SSE2-NEXT: movdqa %xmm6, %xmm11 +; SSE2-NEXT: pxor %xmm9, %xmm11 +; SSE2-NEXT: movdqa %xmm11, %xmm12 +; SSE2-NEXT: pcmpgtd %xmm10, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm10, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm11[1,1,3,3] +; SSE2-NEXT: pand %xmm13, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm12[1,1,3,3] +; SSE2-NEXT: por %xmm11, %xmm10 +; SSE2-NEXT: movdqa %xmm1, %xmm11 +; SSE2-NEXT: pxor %xmm9, %xmm11 +; SSE2-NEXT: movdqa %xmm5, %xmm12 +; SSE2-NEXT: pxor %xmm9, %xmm12 +; SSE2-NEXT: movdqa %xmm12, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm11, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm12[1,1,3,3] +; SSE2-NEXT: pand %xmm14, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm13[1,1,3,3] +; SSE2-NEXT: por %xmm11, %xmm12 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pxor %xmm9, %xmm11 +; SSE2-NEXT: pxor %xmm4, %xmm9 +; SSE2-NEXT: movdqa %xmm9, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm11, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3] +; SSE2-NEXT: pand %xmm14, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[1,1,3,3] +; SSE2-NEXT: por %xmm9, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm11 +; SSE2-NEXT: por %xmm11, %xmm0 +; SSE2-NEXT: pand %xmm12, %xmm1 +; SSE2-NEXT: pandn %xmm5, %xmm12 +; SSE2-NEXT: por %xmm12, %xmm1 +; SSE2-NEXT: pand %xmm10, %xmm2 +; SSE2-NEXT: pandn %xmm6, %xmm10 +; SSE2-NEXT: por %xmm10, %xmm2 +; SSE2-NEXT: pand %xmm8, %xmm3 +; SSE2-NEXT: pandn %xmm7, %xmm8 +; SSE2-NEXT: por %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test125: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm8 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm3, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: movdqa %xmm7, %xmm9 +; SSE4-NEXT: pxor %xmm0, %xmm9 +; SSE4-NEXT: pcmpgtq %xmm10, %xmm9 +; SSE4-NEXT: movdqa %xmm2, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: movdqa %xmm6, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: pcmpgtq %xmm11, %xmm10 +; SSE4-NEXT: movdqa %xmm1, %xmm12 +; SSE4-NEXT: pxor %xmm0, %xmm12 +; SSE4-NEXT: movdqa %xmm5, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm12, %xmm11 +; SSE4-NEXT: movdqa %xmm8, %xmm12 +; SSE4-NEXT: pxor %xmm0, %xmm12 +; SSE4-NEXT: pxor %xmm4, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm12, %xmm0 +; SSE4-NEXT: blendvpd %xmm8, %xmm4 +; SSE4-NEXT: movdqa %xmm11, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm5 +; SSE4-NEXT: movdqa %xmm10, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm6 +; SSE4-NEXT: movdqa %xmm9, %xmm0 +; SSE4-NEXT: blendvpd %xmm3, %xmm7 +; SSE4-NEXT: movapd %xmm4, %xmm0 +; SSE4-NEXT: movapd %xmm5, %xmm1 +; SSE4-NEXT: movapd %xmm6, %xmm2 +; SSE4-NEXT: movapd %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test125: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vmovaps {{.*#+}} xmm5 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm5, %xmm4, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm6, %xmm4 +; AVX1-NEXT: vxorps %xmm5, %xmm1, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm3, %xmm7 +; AVX1-NEXT: vpcmpgtq %xmm6, %xmm7, %xmm6 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm7, %xmm7 +; AVX1-NEXT: vpcmpgtq %xmm6, %xmm7, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm0, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm2, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm7, %xmm5, %xmm5 +; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5 +; AVX1-NEXT: vblendvpd %ymm5, %ymm0, %ymm2, %ymm0 +; AVX1-NEXT: vblendvpd %ymm4, %ymm1, %ymm3, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test125: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm4 +; AVX2-NEXT: vpxor %ymm4, %ymm1, %ymm5 +; AVX2-NEXT: vpxor %ymm4, %ymm3, %ymm6 +; AVX2-NEXT: vpcmpgtq %ymm5, %ymm6, %ymm5 +; AVX2-NEXT: vpxor %ymm4, %ymm0, %ymm6 +; AVX2-NEXT: vpxor %ymm4, %ymm2, %ymm4 +; AVX2-NEXT: vpcmpgtq %ymm6, %ymm4, %ymm4 +; AVX2-NEXT: vblendvpd %ymm4, %ymm0, %ymm2, %ymm0 +; AVX2-NEXT: vblendvpd %ymm5, %ymm1, %ymm3, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test125: -; AVX512F: vpminuq {{.*}} -} - -define void @test126(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i64>* - %ptr.b = bitcast i32* %gep.b to <8 x i64>* - %load.a = load <8 x i64>, <8 x i64>* %ptr.a, align 2 - %load.b = load <8 x i64>, <8 x i64>* %ptr.b, align 2 - %cmp = icmp ule <8 x i64> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i64> %load.a, <8 x i64> %load.b - store <8 x i64> %sel, <8 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminuq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ult <8 x i64> %a, %b + %sel = select <8 x i1> %cmp, <8 x i64> %a, <8 x i64> %b + ret <8 x i64> %sel +} + +define <8 x i64> @test126(<8 x i64> %a, <8 x i64> %b) { +; SSE2-LABEL: test126: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: movdqa %xmm8, -{{[0-9]+}}(%rsp) # 16-byte Spill +; SSE2-NEXT: movdqa %xmm3, %xmm7 +; SSE2-NEXT: movdqa %xmm2, %xmm3 +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm0, %xmm9 +; SSE2-NEXT: movdqa {{.*#+}} xmm10 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: pxor %xmm10, %xmm8 +; SSE2-NEXT: movdqa %xmm7, %xmm0 +; SSE2-NEXT: pxor %xmm10, %xmm0 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm8, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE2-NEXT: pand %xmm12, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: movdqa %xmm12, %xmm8 +; SSE2-NEXT: pxor %xmm1, %xmm8 +; SSE2-NEXT: movdqa %xmm6, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm3, %xmm13 +; SSE2-NEXT: pxor %xmm10, %xmm13 +; SSE2-NEXT: movdqa %xmm13, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm11, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[1,1,3,3] +; SSE2-NEXT: pand %xmm15, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm14[1,1,3,3] +; SSE2-NEXT: por %xmm11, %xmm13 +; SSE2-NEXT: movdqa %xmm5, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm2, %xmm14 +; SSE2-NEXT: pxor %xmm10, %xmm14 +; SSE2-NEXT: movdqa %xmm14, %xmm15 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm15 +; SSE2-NEXT: pcmpeqd %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm15[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm14[1,1,3,3] +; SSE2-NEXT: pand %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm15[1,1,3,3] +; SSE2-NEXT: por %xmm14, %xmm15 +; SSE2-NEXT: movdqa %xmm4, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: pxor %xmm9, %xmm10 +; SSE2-NEXT: movdqa %xmm10, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: pcmpeqd %xmm11, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm14[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm13, %xmm10 +; SSE2-NEXT: pxor %xmm1, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm14[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm14 +; SSE2-NEXT: movdqa %xmm15, %xmm11 +; SSE2-NEXT: pxor %xmm1, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm1 +; SSE2-NEXT: pandn %xmm9, %xmm14 +; SSE2-NEXT: pandn %xmm4, %xmm1 +; SSE2-NEXT: por %xmm14, %xmm1 +; SSE2-NEXT: pandn %xmm2, %xmm15 +; SSE2-NEXT: pandn %xmm5, %xmm11 +; SSE2-NEXT: por %xmm15, %xmm11 +; SSE2-NEXT: pandn %xmm3, %xmm13 +; SSE2-NEXT: pandn %xmm6, %xmm10 +; SSE2-NEXT: por %xmm13, %xmm10 +; SSE2-NEXT: pandn %xmm7, %xmm12 +; SSE2-NEXT: pandn -{{[0-9]+}}(%rsp), %xmm8 # 16-byte Folded Reload +; SSE2-NEXT: por %xmm12, %xmm8 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: movdqa %xmm11, %xmm1 +; SSE2-NEXT: movdqa %xmm10, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test126: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm9 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm7, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: movdqa %xmm3, %xmm8 +; SSE4-NEXT: pxor %xmm0, %xmm8 +; SSE4-NEXT: pcmpgtq %xmm10, %xmm8 +; SSE4-NEXT: pcmpeqd %xmm12, %xmm12 +; SSE4-NEXT: pxor %xmm12, %xmm8 +; SSE4-NEXT: movdqa %xmm6, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: movdqa %xmm2, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: pcmpgtq %xmm11, %xmm10 +; SSE4-NEXT: pxor %xmm12, %xmm10 +; SSE4-NEXT: movdqa %xmm5, %xmm13 +; SSE4-NEXT: pxor %xmm0, %xmm13 +; SSE4-NEXT: movdqa %xmm1, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm13, %xmm11 +; SSE4-NEXT: pxor %xmm12, %xmm11 +; SSE4-NEXT: movdqa %xmm4, %xmm13 +; SSE4-NEXT: pxor %xmm0, %xmm13 +; SSE4-NEXT: pxor %xmm9, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm13, %xmm0 +; SSE4-NEXT: pxor %xmm12, %xmm0 +; SSE4-NEXT: blendvpd %xmm9, %xmm4 +; SSE4-NEXT: movdqa %xmm11, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm5 +; SSE4-NEXT: movdqa %xmm10, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm6 +; SSE4-NEXT: movdqa %xmm8, %xmm0 +; SSE4-NEXT: blendvpd %xmm3, %xmm7 +; SSE4-NEXT: movapd %xmm4, %xmm0 +; SSE4-NEXT: movapd %xmm5, %xmm1 +; SSE4-NEXT: movapd %xmm6, %xmm2 +; SSE4-NEXT: movapd %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test126: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4 +; AVX1-NEXT: vmovaps {{.*#+}} xmm5 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm5, %xmm4, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm6, %xmm4 +; AVX1-NEXT: vpcmpeqd %xmm8, %xmm8, %xmm8 +; AVX1-NEXT: vpxor %xmm8, %xmm4, %xmm4 +; AVX1-NEXT: vxorps %xmm5, %xmm3, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm1, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm7, %xmm6, %xmm6 +; AVX1-NEXT: vpxor %xmm8, %xmm6, %xmm6 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4 +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm7, %xmm7 +; AVX1-NEXT: vpcmpgtq %xmm6, %xmm7, %xmm6 +; AVX1-NEXT: vpxor %xmm8, %xmm6, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm2, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm0, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm7, %xmm5, %xmm5 +; AVX1-NEXT: vpxor %xmm8, %xmm5, %xmm5 +; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5 +; AVX1-NEXT: vblendvpd %ymm5, %ymm0, %ymm2, %ymm0 +; AVX1-NEXT: vblendvpd %ymm4, %ymm1, %ymm3, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test126: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm4 +; AVX2-NEXT: vpxor %ymm4, %ymm3, %ymm5 +; AVX2-NEXT: vpxor %ymm4, %ymm1, %ymm6 +; AVX2-NEXT: vpcmpgtq %ymm5, %ymm6, %ymm5 +; AVX2-NEXT: vpcmpeqd %ymm6, %ymm6, %ymm6 +; AVX2-NEXT: vpxor %ymm6, %ymm5, %ymm5 +; AVX2-NEXT: vpxor %ymm4, %ymm2, %ymm7 +; AVX2-NEXT: vpxor %ymm4, %ymm0, %ymm4 +; AVX2-NEXT: vpcmpgtq %ymm7, %ymm4, %ymm4 +; AVX2-NEXT: vpxor %ymm6, %ymm4, %ymm4 +; AVX2-NEXT: vblendvpd %ymm4, %ymm0, %ymm2, %ymm0 +; AVX2-NEXT: vblendvpd %ymm5, %ymm1, %ymm3, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test126: -; AVX512F: vpminuq {{.*}} -} - -define void @test127(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i64>* - %ptr.b = bitcast i32* %gep.b to <8 x i64>* - %load.a = load <8 x i64>, <8 x i64>* %ptr.a, align 2 - %load.b = load <8 x i64>, <8 x i64>* %ptr.b, align 2 - %cmp = icmp ugt <8 x i64> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i64> %load.a, <8 x i64> %load.b - store <8 x i64> %sel, <8 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminuq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ule <8 x i64> %a, %b + %sel = select <8 x i1> %cmp, <8 x i64> %a, <8 x i64> %b + ret <8 x i64> %sel +} + +define <8 x i64> @test127(<8 x i64> %a, <8 x i64> %b) { +; SSE2-LABEL: test127: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: pxor %xmm9, %xmm8 +; SSE2-NEXT: movdqa %xmm3, %xmm10 +; SSE2-NEXT: pxor %xmm9, %xmm10 +; SSE2-NEXT: movdqa %xmm10, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm8, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3] +; SSE2-NEXT: pand %xmm12, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm11[1,1,3,3] +; SSE2-NEXT: por %xmm10, %xmm8 +; SSE2-NEXT: movdqa %xmm6, %xmm10 +; SSE2-NEXT: pxor %xmm9, %xmm10 +; SSE2-NEXT: movdqa %xmm2, %xmm11 +; SSE2-NEXT: pxor %xmm9, %xmm11 +; SSE2-NEXT: movdqa %xmm11, %xmm12 +; SSE2-NEXT: pcmpgtd %xmm10, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm10, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm11[1,1,3,3] +; SSE2-NEXT: pand %xmm13, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm12[1,1,3,3] +; SSE2-NEXT: por %xmm11, %xmm10 +; SSE2-NEXT: movdqa %xmm5, %xmm11 +; SSE2-NEXT: pxor %xmm9, %xmm11 +; SSE2-NEXT: movdqa %xmm1, %xmm12 +; SSE2-NEXT: pxor %xmm9, %xmm12 +; SSE2-NEXT: movdqa %xmm12, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm11, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm12[1,1,3,3] +; SSE2-NEXT: pand %xmm14, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm13[1,1,3,3] +; SSE2-NEXT: por %xmm11, %xmm12 +; SSE2-NEXT: movdqa %xmm4, %xmm11 +; SSE2-NEXT: pxor %xmm9, %xmm11 +; SSE2-NEXT: pxor %xmm0, %xmm9 +; SSE2-NEXT: movdqa %xmm9, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm11, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3] +; SSE2-NEXT: pand %xmm14, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[1,1,3,3] +; SSE2-NEXT: por %xmm9, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm11 +; SSE2-NEXT: por %xmm11, %xmm0 +; SSE2-NEXT: pand %xmm12, %xmm1 +; SSE2-NEXT: pandn %xmm5, %xmm12 +; SSE2-NEXT: por %xmm12, %xmm1 +; SSE2-NEXT: pand %xmm10, %xmm2 +; SSE2-NEXT: pandn %xmm6, %xmm10 +; SSE2-NEXT: por %xmm10, %xmm2 +; SSE2-NEXT: pand %xmm8, %xmm3 +; SSE2-NEXT: pandn %xmm7, %xmm8 +; SSE2-NEXT: por %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test127: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm8 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm7, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: movdqa %xmm3, %xmm9 +; SSE4-NEXT: pxor %xmm0, %xmm9 +; SSE4-NEXT: pcmpgtq %xmm10, %xmm9 +; SSE4-NEXT: movdqa %xmm6, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: movdqa %xmm2, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: pcmpgtq %xmm11, %xmm10 +; SSE4-NEXT: movdqa %xmm5, %xmm12 +; SSE4-NEXT: pxor %xmm0, %xmm12 +; SSE4-NEXT: movdqa %xmm1, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm12, %xmm11 +; SSE4-NEXT: movdqa %xmm4, %xmm12 +; SSE4-NEXT: pxor %xmm0, %xmm12 +; SSE4-NEXT: pxor %xmm8, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm12, %xmm0 +; SSE4-NEXT: blendvpd %xmm8, %xmm4 +; SSE4-NEXT: movdqa %xmm11, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm5 +; SSE4-NEXT: movdqa %xmm10, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm6 +; SSE4-NEXT: movdqa %xmm9, %xmm0 +; SSE4-NEXT: blendvpd %xmm3, %xmm7 +; SSE4-NEXT: movapd %xmm4, %xmm0 +; SSE4-NEXT: movapd %xmm5, %xmm1 +; SSE4-NEXT: movapd %xmm6, %xmm2 +; SSE4-NEXT: movapd %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test127: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4 +; AVX1-NEXT: vmovaps {{.*#+}} xmm5 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm5, %xmm4, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm6, %xmm4 +; AVX1-NEXT: vxorps %xmm5, %xmm3, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm1, %xmm7 +; AVX1-NEXT: vpcmpgtq %xmm6, %xmm7, %xmm6 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4 +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm7, %xmm7 +; AVX1-NEXT: vpcmpgtq %xmm6, %xmm7, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm2, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm0, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm7, %xmm5, %xmm5 +; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5 +; AVX1-NEXT: vblendvpd %ymm5, %ymm0, %ymm2, %ymm0 +; AVX1-NEXT: vblendvpd %ymm4, %ymm1, %ymm3, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test127: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm4 +; AVX2-NEXT: vpxor %ymm4, %ymm3, %ymm5 +; AVX2-NEXT: vpxor %ymm4, %ymm1, %ymm6 +; AVX2-NEXT: vpcmpgtq %ymm5, %ymm6, %ymm5 +; AVX2-NEXT: vpxor %ymm4, %ymm2, %ymm6 +; AVX2-NEXT: vpxor %ymm4, %ymm0, %ymm4 +; AVX2-NEXT: vpcmpgtq %ymm6, %ymm4, %ymm4 +; AVX2-NEXT: vblendvpd %ymm4, %ymm0, %ymm2, %ymm0 +; AVX2-NEXT: vblendvpd %ymm5, %ymm1, %ymm3, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test127: -; AVX512F: vpmaxuq {{.*}} -} - -define void @test128(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i64>* - %ptr.b = bitcast i32* %gep.b to <8 x i64>* - %load.a = load <8 x i64>, <8 x i64>* %ptr.a, align 2 - %load.b = load <8 x i64>, <8 x i64>* %ptr.b, align 2 - %cmp = icmp uge <8 x i64> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i64> %load.a, <8 x i64> %load.b - store <8 x i64> %sel, <8 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxuq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ugt <8 x i64> %a, %b + %sel = select <8 x i1> %cmp, <8 x i64> %a, <8 x i64> %b + ret <8 x i64> %sel +} + +define <8 x i64> @test128(<8 x i64> %a, <8 x i64> %b) { +; SSE2-LABEL: test128: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm7, %xmm11 +; SSE2-NEXT: movdqa %xmm11, -{{[0-9]+}}(%rsp) # 16-byte Spill +; SSE2-NEXT: movdqa %xmm3, %xmm7 +; SSE2-NEXT: movdqa %xmm2, %xmm3 +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm0, %xmm9 +; SSE2-NEXT: movdqa {{.*#+}} xmm10 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: pxor %xmm10, %xmm8 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: pxor %xmm10, %xmm0 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm8, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE2-NEXT: pand %xmm12, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: movdqa %xmm12, %xmm8 +; SSE2-NEXT: pxor %xmm1, %xmm8 +; SSE2-NEXT: movdqa %xmm3, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm6, %xmm13 +; SSE2-NEXT: pxor %xmm10, %xmm13 +; SSE2-NEXT: movdqa %xmm13, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm11, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[1,1,3,3] +; SSE2-NEXT: pand %xmm15, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm14[1,1,3,3] +; SSE2-NEXT: por %xmm11, %xmm13 +; SSE2-NEXT: movdqa %xmm2, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm5, %xmm14 +; SSE2-NEXT: pxor %xmm10, %xmm14 +; SSE2-NEXT: movdqa %xmm14, %xmm15 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm15 +; SSE2-NEXT: pcmpeqd %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm15[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm14[1,1,3,3] +; SSE2-NEXT: pand %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm15[1,1,3,3] +; SSE2-NEXT: por %xmm14, %xmm15 +; SSE2-NEXT: movdqa %xmm9, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: pxor %xmm4, %xmm10 +; SSE2-NEXT: movdqa %xmm10, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: pcmpeqd %xmm11, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm14[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm13, %xmm10 +; SSE2-NEXT: pxor %xmm1, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm14[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm14 +; SSE2-NEXT: movdqa %xmm15, %xmm11 +; SSE2-NEXT: pxor %xmm1, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm1 +; SSE2-NEXT: pandn %xmm9, %xmm14 +; SSE2-NEXT: pandn %xmm4, %xmm1 +; SSE2-NEXT: por %xmm14, %xmm1 +; SSE2-NEXT: pandn %xmm2, %xmm15 +; SSE2-NEXT: pandn %xmm5, %xmm11 +; SSE2-NEXT: por %xmm15, %xmm11 +; SSE2-NEXT: pandn %xmm3, %xmm13 +; SSE2-NEXT: pandn %xmm6, %xmm10 +; SSE2-NEXT: por %xmm13, %xmm10 +; SSE2-NEXT: pandn %xmm7, %xmm12 +; SSE2-NEXT: pandn -{{[0-9]+}}(%rsp), %xmm8 # 16-byte Folded Reload +; SSE2-NEXT: por %xmm12, %xmm8 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: movdqa %xmm11, %xmm1 +; SSE2-NEXT: movdqa %xmm10, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test128: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm9 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm3, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: movdqa %xmm7, %xmm8 +; SSE4-NEXT: pxor %xmm0, %xmm8 +; SSE4-NEXT: pcmpgtq %xmm10, %xmm8 +; SSE4-NEXT: pcmpeqd %xmm12, %xmm12 +; SSE4-NEXT: pxor %xmm12, %xmm8 +; SSE4-NEXT: movdqa %xmm2, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: movdqa %xmm6, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: pcmpgtq %xmm11, %xmm10 +; SSE4-NEXT: pxor %xmm12, %xmm10 +; SSE4-NEXT: movdqa %xmm1, %xmm13 +; SSE4-NEXT: pxor %xmm0, %xmm13 +; SSE4-NEXT: movdqa %xmm5, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm13, %xmm11 +; SSE4-NEXT: pxor %xmm12, %xmm11 +; SSE4-NEXT: movdqa %xmm9, %xmm13 +; SSE4-NEXT: pxor %xmm0, %xmm13 +; SSE4-NEXT: pxor %xmm4, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm13, %xmm0 +; SSE4-NEXT: pxor %xmm12, %xmm0 +; SSE4-NEXT: blendvpd %xmm9, %xmm4 +; SSE4-NEXT: movdqa %xmm11, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm5 +; SSE4-NEXT: movdqa %xmm10, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm6 +; SSE4-NEXT: movdqa %xmm8, %xmm0 +; SSE4-NEXT: blendvpd %xmm3, %xmm7 +; SSE4-NEXT: movapd %xmm4, %xmm0 +; SSE4-NEXT: movapd %xmm5, %xmm1 +; SSE4-NEXT: movapd %xmm6, %xmm2 +; SSE4-NEXT: movapd %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test128: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vmovaps {{.*#+}} xmm5 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm5, %xmm4, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm6, %xmm4 +; AVX1-NEXT: vpcmpeqd %xmm8, %xmm8, %xmm8 +; AVX1-NEXT: vpxor %xmm8, %xmm4, %xmm4 +; AVX1-NEXT: vxorps %xmm5, %xmm1, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm3, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm7, %xmm6, %xmm6 +; AVX1-NEXT: vpxor %xmm8, %xmm6, %xmm6 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm7, %xmm7 +; AVX1-NEXT: vpcmpgtq %xmm6, %xmm7, %xmm6 +; AVX1-NEXT: vpxor %xmm8, %xmm6, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm0, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm2, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm7, %xmm5, %xmm5 +; AVX1-NEXT: vpxor %xmm8, %xmm5, %xmm5 +; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5 +; AVX1-NEXT: vblendvpd %ymm5, %ymm0, %ymm2, %ymm0 +; AVX1-NEXT: vblendvpd %ymm4, %ymm1, %ymm3, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test128: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm4 +; AVX2-NEXT: vpxor %ymm4, %ymm1, %ymm5 +; AVX2-NEXT: vpxor %ymm4, %ymm3, %ymm6 +; AVX2-NEXT: vpcmpgtq %ymm5, %ymm6, %ymm5 +; AVX2-NEXT: vpcmpeqd %ymm6, %ymm6, %ymm6 +; AVX2-NEXT: vpxor %ymm6, %ymm5, %ymm5 +; AVX2-NEXT: vpxor %ymm4, %ymm0, %ymm7 +; AVX2-NEXT: vpxor %ymm4, %ymm2, %ymm4 +; AVX2-NEXT: vpcmpgtq %ymm7, %ymm4, %ymm4 +; AVX2-NEXT: vpxor %ymm6, %ymm4, %ymm4 +; AVX2-NEXT: vblendvpd %ymm4, %ymm0, %ymm2, %ymm0 +; AVX2-NEXT: vblendvpd %ymm5, %ymm1, %ymm3, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test128: -; AVX512F: vpmaxuq {{.*}} -} - -define void @test129(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <64 x i8>* - %ptr.b = bitcast i8* %gep.b to <64 x i8>* - %load.a = load <64 x i8>, <64 x i8>* %ptr.a, align 2 - %load.b = load <64 x i8>, <64 x i8>* %ptr.b, align 2 - %cmp = icmp slt <64 x i8> %load.a, %load.b - %sel = select <64 x i1> %cmp, <64 x i8> %load.b, <64 x i8> %load.a - store <64 x i8> %sel, <64 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxuq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp uge <8 x i64> %a, %b + %sel = select <8 x i1> %cmp, <8 x i64> %a, <8 x i64> %b + ret <8 x i64> %sel +} + +define <64 x i8> @test129(<64 x i8> %a, <64 x i8> %b) { +; SSE2-LABEL: test129: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: movdqa %xmm7, %xmm3 +; SSE2-NEXT: pcmpgtb %xmm8, %xmm3 +; SSE2-NEXT: movdqa %xmm6, %xmm9 +; SSE2-NEXT: pcmpgtb %xmm2, %xmm9 +; SSE2-NEXT: movdqa %xmm5, %xmm10 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm10 +; SSE2-NEXT: movdqa %xmm4, %xmm11 +; SSE2-NEXT: pcmpgtb %xmm0, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm11 +; SSE2-NEXT: por %xmm4, %xmm11 +; SSE2-NEXT: pand %xmm10, %xmm5 +; SSE2-NEXT: pandn %xmm1, %xmm10 +; SSE2-NEXT: por %xmm5, %xmm10 +; SSE2-NEXT: pand %xmm9, %xmm6 +; SSE2-NEXT: pandn %xmm2, %xmm9 +; SSE2-NEXT: por %xmm6, %xmm9 +; SSE2-NEXT: pand %xmm3, %xmm7 +; SSE2-NEXT: pandn %xmm8, %xmm3 +; SSE2-NEXT: por %xmm7, %xmm3 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test129: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsb %xmm4, %xmm0 +; SSE4-NEXT: pmaxsb %xmm5, %xmm1 +; SSE4-NEXT: pmaxsb %xmm6, %xmm2 +; SSE4-NEXT: pmaxsb %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test129: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxsb %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxsb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxsb %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxsb %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test129: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsb %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxsb %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test129: -; AVX512BW: vpmaxsb -} - -define void @test130(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <64 x i8>* - %ptr.b = bitcast i8* %gep.b to <64 x i8>* - %load.a = load <64 x i8>, <64 x i8>* %ptr.a, align 2 - %load.b = load <64 x i8>, <64 x i8>* %ptr.b, align 2 - %cmp = icmp sle <64 x i8> %load.a, %load.b - %sel = select <64 x i1> %cmp, <64 x i8> %load.b, <64 x i8> %load.a - store <64 x i8> %sel, <64 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxsb %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp slt <64 x i8> %a, %b + %sel = select <64 x i1> %cmp, <64 x i8> %b, <64 x i8> %a + ret <64 x i8> %sel +} + +define <64 x i8> @test130(<64 x i8> %a, <64 x i8> %b) { +; SSE2-LABEL: test130: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm2, %xmm8 +; SSE2-NEXT: movdqa %xmm3, %xmm12 +; SSE2-NEXT: pcmpgtb %xmm7, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm13, %xmm13 +; SSE2-NEXT: movdqa %xmm12, %xmm9 +; SSE2-NEXT: pxor %xmm13, %xmm9 +; SSE2-NEXT: movdqa %xmm8, %xmm14 +; SSE2-NEXT: pcmpgtb %xmm6, %xmm14 +; SSE2-NEXT: movdqa %xmm14, %xmm2 +; SSE2-NEXT: pxor %xmm13, %xmm2 +; SSE2-NEXT: movdqa %xmm1, %xmm15 +; SSE2-NEXT: pcmpgtb %xmm5, %xmm15 +; SSE2-NEXT: movdqa %xmm15, %xmm10 +; SSE2-NEXT: pxor %xmm13, %xmm10 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtb %xmm4, %xmm11 +; SSE2-NEXT: pxor %xmm11, %xmm13 +; SSE2-NEXT: pandn %xmm4, %xmm11 +; SSE2-NEXT: pandn %xmm0, %xmm13 +; SSE2-NEXT: por %xmm13, %xmm11 +; SSE2-NEXT: pandn %xmm5, %xmm15 +; SSE2-NEXT: pandn %xmm1, %xmm10 +; SSE2-NEXT: por %xmm15, %xmm10 +; SSE2-NEXT: pandn %xmm6, %xmm14 +; SSE2-NEXT: pandn %xmm8, %xmm2 +; SSE2-NEXT: por %xmm14, %xmm2 +; SSE2-NEXT: pandn %xmm7, %xmm12 +; SSE2-NEXT: pandn %xmm3, %xmm9 +; SSE2-NEXT: por %xmm12, %xmm9 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test130: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsb %xmm4, %xmm0 +; SSE4-NEXT: pmaxsb %xmm5, %xmm1 +; SSE4-NEXT: pmaxsb %xmm6, %xmm2 +; SSE4-NEXT: pmaxsb %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test130: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxsb %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxsb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxsb %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxsb %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test130: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsb %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxsb %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test130: -; AVX512BW: vpmaxsb -} - -define void @test131(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <64 x i8>* - %ptr.b = bitcast i8* %gep.b to <64 x i8>* - %load.a = load <64 x i8>, <64 x i8>* %ptr.a, align 2 - %load.b = load <64 x i8>, <64 x i8>* %ptr.b, align 2 - %cmp = icmp sgt <64 x i8> %load.a, %load.b - %sel = select <64 x i1> %cmp, <64 x i8> %load.b, <64 x i8> %load.a - store <64 x i8> %sel, <64 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxsb %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sle <64 x i8> %a, %b + %sel = select <64 x i1> %cmp, <64 x i8> %b, <64 x i8> %a + ret <64 x i8> %sel +} + +define <64 x i8> @test131(<64 x i8> %a, <64 x i8> %b) { +; SSE2-LABEL: test131: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: pcmpgtb %xmm7, %xmm3 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: pcmpgtb %xmm6, %xmm9 +; SSE2-NEXT: movdqa %xmm1, %xmm10 +; SSE2-NEXT: pcmpgtb %xmm5, %xmm10 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtb %xmm4, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm11 +; SSE2-NEXT: por %xmm4, %xmm11 +; SSE2-NEXT: pand %xmm10, %xmm5 +; SSE2-NEXT: pandn %xmm1, %xmm10 +; SSE2-NEXT: por %xmm5, %xmm10 +; SSE2-NEXT: pand %xmm9, %xmm6 +; SSE2-NEXT: pandn %xmm2, %xmm9 +; SSE2-NEXT: por %xmm6, %xmm9 +; SSE2-NEXT: pand %xmm3, %xmm7 +; SSE2-NEXT: pandn %xmm8, %xmm3 +; SSE2-NEXT: por %xmm7, %xmm3 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test131: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsb %xmm4, %xmm0 +; SSE4-NEXT: pminsb %xmm5, %xmm1 +; SSE4-NEXT: pminsb %xmm6, %xmm2 +; SSE4-NEXT: pminsb %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test131: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminsb %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminsb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminsb %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminsb %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test131: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsb %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminsb %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test131: -; AVX512BW: vpminsb -} - -define void @test132(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <64 x i8>* - %ptr.b = bitcast i8* %gep.b to <64 x i8>* - %load.a = load <64 x i8>, <64 x i8>* %ptr.a, align 2 - %load.b = load <64 x i8>, <64 x i8>* %ptr.b, align 2 - %cmp = icmp sge <64 x i8> %load.a, %load.b - %sel = select <64 x i1> %cmp, <64 x i8> %load.b, <64 x i8> %load.a - store <64 x i8> %sel, <64 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminsb %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sgt <64 x i8> %a, %b + %sel = select <64 x i1> %cmp, <64 x i8> %b, <64 x i8> %a + ret <64 x i8> %sel +} + +define <64 x i8> @test132(<64 x i8> %a, <64 x i8> %b) { +; SSE2-LABEL: test132: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm2, %xmm8 +; SSE2-NEXT: movdqa %xmm0, %xmm10 +; SSE2-NEXT: movdqa %xmm7, %xmm12 +; SSE2-NEXT: pcmpgtb %xmm3, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE2-NEXT: movdqa %xmm12, %xmm9 +; SSE2-NEXT: pxor %xmm0, %xmm9 +; SSE2-NEXT: movdqa %xmm6, %xmm13 +; SSE2-NEXT: pcmpgtb %xmm8, %xmm13 +; SSE2-NEXT: movdqa %xmm13, %xmm2 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm5, %xmm14 +; SSE2-NEXT: pcmpgtb %xmm1, %xmm14 +; SSE2-NEXT: movdqa %xmm14, %xmm11 +; SSE2-NEXT: pxor %xmm0, %xmm11 +; SSE2-NEXT: movdqa %xmm4, %xmm15 +; SSE2-NEXT: pcmpgtb %xmm10, %xmm15 +; SSE2-NEXT: pxor %xmm15, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm15 +; SSE2-NEXT: pandn %xmm10, %xmm0 +; SSE2-NEXT: por %xmm15, %xmm0 +; SSE2-NEXT: pandn %xmm5, %xmm14 +; SSE2-NEXT: pandn %xmm1, %xmm11 +; SSE2-NEXT: por %xmm14, %xmm11 +; SSE2-NEXT: pandn %xmm6, %xmm13 +; SSE2-NEXT: pandn %xmm8, %xmm2 +; SSE2-NEXT: por %xmm13, %xmm2 +; SSE2-NEXT: pandn %xmm7, %xmm12 +; SSE2-NEXT: pandn %xmm3, %xmm9 +; SSE2-NEXT: por %xmm12, %xmm9 +; SSE2-NEXT: movdqa %xmm11, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test132: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsb %xmm4, %xmm0 +; SSE4-NEXT: pminsb %xmm5, %xmm1 +; SSE4-NEXT: pminsb %xmm6, %xmm2 +; SSE4-NEXT: pminsb %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test132: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminsb %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminsb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminsb %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminsb %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test132: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsb %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminsb %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test132: -; AVX512BW: vpminsb -} - -define void @test133(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <64 x i8>* - %ptr.b = bitcast i8* %gep.b to <64 x i8>* - %load.a = load <64 x i8>, <64 x i8>* %ptr.a, align 2 - %load.b = load <64 x i8>, <64 x i8>* %ptr.b, align 2 - %cmp = icmp ult <64 x i8> %load.a, %load.b - %sel = select <64 x i1> %cmp, <64 x i8> %load.b, <64 x i8> %load.a - store <64 x i8> %sel, <64 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminsb %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sge <64 x i8> %a, %b + %sel = select <64 x i1> %cmp, <64 x i8> %b, <64 x i8> %a + ret <64 x i8> %sel +} + +define <64 x i8> @test133(<64 x i8> %a, <64 x i8> %b) { +; SSE-LABEL: test133: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxub %xmm4, %xmm0 +; SSE-NEXT: pmaxub %xmm5, %xmm1 +; SSE-NEXT: pmaxub %xmm6, %xmm2 +; SSE-NEXT: pmaxub %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX1-LABEL: test133: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxub %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxub %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxub %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxub %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test133: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxub %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxub %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test133: -; AVX512BW: vpmaxub -} - -define void @test134(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <64 x i8>* - %ptr.b = bitcast i8* %gep.b to <64 x i8>* - %load.a = load <64 x i8>, <64 x i8>* %ptr.a, align 2 - %load.b = load <64 x i8>, <64 x i8>* %ptr.b, align 2 - %cmp = icmp ule <64 x i8> %load.a, %load.b - %sel = select <64 x i1> %cmp, <64 x i8> %load.b, <64 x i8> %load.a - store <64 x i8> %sel, <64 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxub %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ult <64 x i8> %a, %b + %sel = select <64 x i1> %cmp, <64 x i8> %b, <64 x i8> %a + ret <64 x i8> %sel +} + +define <64 x i8> @test134(<64 x i8> %a, <64 x i8> %b) { +; SSE-LABEL: test134: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxub %xmm4, %xmm0 +; SSE-NEXT: pmaxub %xmm5, %xmm1 +; SSE-NEXT: pmaxub %xmm6, %xmm2 +; SSE-NEXT: pmaxub %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX1-LABEL: test134: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxub %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxub %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxub %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxub %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test134: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxub %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxub %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test134: -; AVX512BW: vpmaxub -} - -define void @test135(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <64 x i8>* - %ptr.b = bitcast i8* %gep.b to <64 x i8>* - %load.a = load <64 x i8>, <64 x i8>* %ptr.a, align 2 - %load.b = load <64 x i8>, <64 x i8>* %ptr.b, align 2 - %cmp = icmp ugt <64 x i8> %load.a, %load.b - %sel = select <64 x i1> %cmp, <64 x i8> %load.b, <64 x i8> %load.a - store <64 x i8> %sel, <64 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxub %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ule <64 x i8> %a, %b + %sel = select <64 x i1> %cmp, <64 x i8> %b, <64 x i8> %a + ret <64 x i8> %sel +} + +define <64 x i8> @test135(<64 x i8> %a, <64 x i8> %b) { +; SSE-LABEL: test135: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminub %xmm4, %xmm0 +; SSE-NEXT: pminub %xmm5, %xmm1 +; SSE-NEXT: pminub %xmm6, %xmm2 +; SSE-NEXT: pminub %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX1-LABEL: test135: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminub %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminub %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminub %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminub %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test135: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminub %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminub %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test135: -; AVX512BW: vpminub -} - -define void @test136(i8* nocapture %a, i8* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i8, i8* %a, i64 %index - %gep.b = getelementptr inbounds i8, i8* %b, i64 %index - %ptr.a = bitcast i8* %gep.a to <64 x i8>* - %ptr.b = bitcast i8* %gep.b to <64 x i8>* - %load.a = load <64 x i8>, <64 x i8>* %ptr.a, align 2 - %load.b = load <64 x i8>, <64 x i8>* %ptr.b, align 2 - %cmp = icmp uge <64 x i8> %load.a, %load.b - %sel = select <64 x i1> %cmp, <64 x i8> %load.b, <64 x i8> %load.a - store <64 x i8> %sel, <64 x i8>* %ptr.a, align 2 - %index.next = add i64 %index, 32 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminub %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ugt <64 x i8> %a, %b + %sel = select <64 x i1> %cmp, <64 x i8> %b, <64 x i8> %a + ret <64 x i8> %sel +} + +define <64 x i8> @test136(<64 x i8> %a, <64 x i8> %b) { +; SSE-LABEL: test136: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminub %xmm4, %xmm0 +; SSE-NEXT: pminub %xmm5, %xmm1 +; SSE-NEXT: pminub %xmm6, %xmm2 +; SSE-NEXT: pminub %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX1-LABEL: test136: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminub %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminub %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminub %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminub %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test136: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminub %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminub %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test136: -; AVX512BW: vpminub -} - -define void @test137(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <32 x i16>* - %ptr.b = bitcast i16* %gep.b to <32 x i16>* - %load.a = load <32 x i16>, <32 x i16>* %ptr.a, align 2 - %load.b = load <32 x i16>, <32 x i16>* %ptr.b, align 2 - %cmp = icmp slt <32 x i16> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i16> %load.b, <32 x i16> %load.a - store <32 x i16> %sel, <32 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminub %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp uge <64 x i8> %a, %b + %sel = select <64 x i1> %cmp, <64 x i8> %b, <64 x i8> %a + ret <64 x i8> %sel +} + +define <32 x i16> @test137(<32 x i16> %a, <32 x i16> %b) { +; SSE-LABEL: test137: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxsw %xmm4, %xmm0 +; SSE-NEXT: pmaxsw %xmm5, %xmm1 +; SSE-NEXT: pmaxsw %xmm6, %xmm2 +; SSE-NEXT: pmaxsw %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX1-LABEL: test137: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxsw %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxsw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxsw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxsw %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test137: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxsw %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test137: -; AVX512BW: vpmaxsw -} - -define void @test138(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <32 x i16>* - %ptr.b = bitcast i16* %gep.b to <32 x i16>* - %load.a = load <32 x i16>, <32 x i16>* %ptr.a, align 2 - %load.b = load <32 x i16>, <32 x i16>* %ptr.b, align 2 - %cmp = icmp sle <32 x i16> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i16> %load.b, <32 x i16> %load.a - store <32 x i16> %sel, <32 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxsw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp slt <32 x i16> %a, %b + %sel = select <32 x i1> %cmp, <32 x i16> %b, <32 x i16> %a + ret <32 x i16> %sel +} + +define <32 x i16> @test138(<32 x i16> %a, <32 x i16> %b) { +; SSE-LABEL: test138: +; SSE: # BB#0: # %entry +; SSE-NEXT: pmaxsw %xmm4, %xmm0 +; SSE-NEXT: pmaxsw %xmm5, %xmm1 +; SSE-NEXT: pmaxsw %xmm6, %xmm2 +; SSE-NEXT: pmaxsw %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX1-LABEL: test138: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxsw %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxsw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxsw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxsw %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test138: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxsw %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test138: -; AVX512BW: vpmaxsw -} - -define void @test139(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <32 x i16>* - %ptr.b = bitcast i16* %gep.b to <32 x i16>* - %load.a = load <32 x i16>, <32 x i16>* %ptr.a, align 2 - %load.b = load <32 x i16>, <32 x i16>* %ptr.b, align 2 - %cmp = icmp sgt <32 x i16> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i16> %load.b, <32 x i16> %load.a - store <32 x i16> %sel, <32 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxsw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sle <32 x i16> %a, %b + %sel = select <32 x i1> %cmp, <32 x i16> %b, <32 x i16> %a + ret <32 x i16> %sel +} + +define <32 x i16> @test139(<32 x i16> %a, <32 x i16> %b) { +; SSE-LABEL: test139: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminsw %xmm4, %xmm0 +; SSE-NEXT: pminsw %xmm5, %xmm1 +; SSE-NEXT: pminsw %xmm6, %xmm2 +; SSE-NEXT: pminsw %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX1-LABEL: test139: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminsw %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminsw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminsw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminsw %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test139: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminsw %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test139: -; AVX512BW: vpminsw -} - -define void @test140(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <32 x i16>* - %ptr.b = bitcast i16* %gep.b to <32 x i16>* - %load.a = load <32 x i16>, <32 x i16>* %ptr.a, align 2 - %load.b = load <32 x i16>, <32 x i16>* %ptr.b, align 2 - %cmp = icmp sge <32 x i16> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i16> %load.b, <32 x i16> %load.a - store <32 x i16> %sel, <32 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminsw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sgt <32 x i16> %a, %b + %sel = select <32 x i1> %cmp, <32 x i16> %b, <32 x i16> %a + ret <32 x i16> %sel +} + +define <32 x i16> @test140(<32 x i16> %a, <32 x i16> %b) { +; SSE-LABEL: test140: +; SSE: # BB#0: # %entry +; SSE-NEXT: pminsw %xmm4, %xmm0 +; SSE-NEXT: pminsw %xmm5, %xmm1 +; SSE-NEXT: pminsw %xmm6, %xmm2 +; SSE-NEXT: pminsw %xmm7, %xmm3 +; SSE-NEXT: retq +; +; AVX1-LABEL: test140: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminsw %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminsw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminsw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminsw %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test140: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminsw %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test140: -; AVX512BW: vpminsw -} - -define void @test141(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <32 x i16>* - %ptr.b = bitcast i16* %gep.b to <32 x i16>* - %load.a = load <32 x i16>, <32 x i16>* %ptr.a, align 2 - %load.b = load <32 x i16>, <32 x i16>* %ptr.b, align 2 - %cmp = icmp ult <32 x i16> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i16> %load.b, <32 x i16> %load.a - store <32 x i16> %sel, <32 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminsw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sge <32 x i16> %a, %b + %sel = select <32 x i1> %cmp, <32 x i16> %b, <32 x i16> %a + ret <32 x i16> %sel +} + +define <32 x i16> @test141(<32 x i16> %a, <32 x i16> %b) { +; SSE2-LABEL: test141: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE2-NEXT: movdqa %xmm3, %xmm9 +; SSE2-NEXT: pxor %xmm0, %xmm9 +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: pxor %xmm0, %xmm8 +; SSE2-NEXT: pcmpgtw %xmm9, %xmm8 +; SSE2-NEXT: movdqa %xmm2, %xmm10 +; SSE2-NEXT: pxor %xmm0, %xmm10 +; SSE2-NEXT: movdqa %xmm6, %xmm9 +; SSE2-NEXT: pxor %xmm0, %xmm9 +; SSE2-NEXT: pcmpgtw %xmm10, %xmm9 +; SSE2-NEXT: movdqa %xmm1, %xmm12 +; SSE2-NEXT: pxor %xmm0, %xmm12 +; SSE2-NEXT: movdqa %xmm5, %xmm10 +; SSE2-NEXT: pxor %xmm0, %xmm10 +; SSE2-NEXT: pcmpgtw %xmm12, %xmm10 +; SSE2-NEXT: movdqa %xmm11, %xmm12 +; SSE2-NEXT: pxor %xmm0, %xmm12 +; SSE2-NEXT: pxor %xmm4, %xmm0 +; SSE2-NEXT: pcmpgtw %xmm12, %xmm0 +; SSE2-NEXT: pand %xmm0, %xmm4 +; SSE2-NEXT: pandn %xmm11, %xmm0 +; SSE2-NEXT: por %xmm4, %xmm0 +; SSE2-NEXT: pand %xmm10, %xmm5 +; SSE2-NEXT: pandn %xmm1, %xmm10 +; SSE2-NEXT: por %xmm5, %xmm10 +; SSE2-NEXT: pand %xmm9, %xmm6 +; SSE2-NEXT: pandn %xmm2, %xmm9 +; SSE2-NEXT: por %xmm6, %xmm9 +; SSE2-NEXT: pand %xmm8, %xmm7 +; SSE2-NEXT: pandn %xmm3, %xmm8 +; SSE2-NEXT: por %xmm7, %xmm8 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test141: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxuw %xmm4, %xmm0 +; SSE4-NEXT: pmaxuw %xmm5, %xmm1 +; SSE4-NEXT: pmaxuw %xmm6, %xmm2 +; SSE4-NEXT: pmaxuw %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test141: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxuw %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxuw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxuw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxuw %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test141: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxuw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxuw %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test141: -; AVX512BW: vpmaxuw -} - -define void @test142(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <32 x i16>* - %ptr.b = bitcast i16* %gep.b to <32 x i16>* - %load.a = load <32 x i16>, <32 x i16>* %ptr.a, align 2 - %load.b = load <32 x i16>, <32 x i16>* %ptr.b, align 2 - %cmp = icmp ule <32 x i16> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i16> %load.b, <32 x i16> %load.a - store <32 x i16> %sel, <32 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxuw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ult <32 x i16> %a, %b + %sel = select <32 x i1> %cmp, <32 x i16> %b, <32 x i16> %a + ret <32 x i16> %sel +} + +define <32 x i16> @test142(<32 x i16> %a, <32 x i16> %b) { +; SSE2-LABEL: test142: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: movdqa %xmm1, %xmm10 +; SSE2-NEXT: psubusw %xmm7, %xmm3 +; SSE2-NEXT: pxor %xmm12, %xmm12 +; SSE2-NEXT: pcmpeqw %xmm12, %xmm3 +; SSE2-NEXT: psubusw %xmm6, %xmm2 +; SSE2-NEXT: pcmpeqw %xmm12, %xmm2 +; SSE2-NEXT: psubusw %xmm5, %xmm1 +; SSE2-NEXT: pcmpeqw %xmm12, %xmm1 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: psubusw %xmm4, %xmm11 +; SSE2-NEXT: pcmpeqw %xmm12, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm11 +; SSE2-NEXT: por %xmm4, %xmm11 +; SSE2-NEXT: pand %xmm1, %xmm5 +; SSE2-NEXT: pandn %xmm10, %xmm1 +; SSE2-NEXT: por %xmm5, %xmm1 +; SSE2-NEXT: pand %xmm2, %xmm6 +; SSE2-NEXT: pandn %xmm9, %xmm2 +; SSE2-NEXT: por %xmm6, %xmm2 +; SSE2-NEXT: pand %xmm3, %xmm7 +; SSE2-NEXT: pandn %xmm8, %xmm3 +; SSE2-NEXT: por %xmm7, %xmm3 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test142: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxuw %xmm4, %xmm0 +; SSE4-NEXT: pmaxuw %xmm5, %xmm1 +; SSE4-NEXT: pmaxuw %xmm6, %xmm2 +; SSE4-NEXT: pmaxuw %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test142: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxuw %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxuw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxuw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxuw %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test142: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxuw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxuw %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test142: -; AVX512BW: vpmaxuw -} - -define void @test143(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <32 x i16>* - %ptr.b = bitcast i16* %gep.b to <32 x i16>* - %load.a = load <32 x i16>, <32 x i16>* %ptr.a, align 2 - %load.b = load <32 x i16>, <32 x i16>* %ptr.b, align 2 - %cmp = icmp ugt <32 x i16> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i16> %load.b, <32 x i16> %load.a - store <32 x i16> %sel, <32 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxuw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ule <32 x i16> %a, %b + %sel = select <32 x i1> %cmp, <32 x i16> %b, <32 x i16> %a + ret <32 x i16> %sel +} + +define <32 x i16> @test143(<32 x i16> %a, <32 x i16> %b) { +; SSE2-LABEL: test143: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE2-NEXT: movdqa %xmm7, %xmm9 +; SSE2-NEXT: pxor %xmm0, %xmm9 +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: pxor %xmm0, %xmm8 +; SSE2-NEXT: pcmpgtw %xmm9, %xmm8 +; SSE2-NEXT: movdqa %xmm6, %xmm10 +; SSE2-NEXT: pxor %xmm0, %xmm10 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: pxor %xmm0, %xmm9 +; SSE2-NEXT: pcmpgtw %xmm10, %xmm9 +; SSE2-NEXT: movdqa %xmm5, %xmm12 +; SSE2-NEXT: pxor %xmm0, %xmm12 +; SSE2-NEXT: movdqa %xmm1, %xmm10 +; SSE2-NEXT: pxor %xmm0, %xmm10 +; SSE2-NEXT: pcmpgtw %xmm12, %xmm10 +; SSE2-NEXT: movdqa %xmm4, %xmm12 +; SSE2-NEXT: pxor %xmm0, %xmm12 +; SSE2-NEXT: pxor %xmm11, %xmm0 +; SSE2-NEXT: pcmpgtw %xmm12, %xmm0 +; SSE2-NEXT: pand %xmm0, %xmm4 +; SSE2-NEXT: pandn %xmm11, %xmm0 +; SSE2-NEXT: por %xmm4, %xmm0 +; SSE2-NEXT: pand %xmm10, %xmm5 +; SSE2-NEXT: pandn %xmm1, %xmm10 +; SSE2-NEXT: por %xmm5, %xmm10 +; SSE2-NEXT: pand %xmm9, %xmm6 +; SSE2-NEXT: pandn %xmm2, %xmm9 +; SSE2-NEXT: por %xmm6, %xmm9 +; SSE2-NEXT: pand %xmm8, %xmm7 +; SSE2-NEXT: pandn %xmm3, %xmm8 +; SSE2-NEXT: por %xmm7, %xmm8 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test143: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminuw %xmm4, %xmm0 +; SSE4-NEXT: pminuw %xmm5, %xmm1 +; SSE4-NEXT: pminuw %xmm6, %xmm2 +; SSE4-NEXT: pminuw %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test143: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminuw %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminuw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminuw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminuw %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test143: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminuw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminuw %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test143: -; AVX512BW: vpminuw -} - -define void @test144(i16* nocapture %a, i16* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i16, i16* %a, i64 %index - %gep.b = getelementptr inbounds i16, i16* %b, i64 %index - %ptr.a = bitcast i16* %gep.a to <32 x i16>* - %ptr.b = bitcast i16* %gep.b to <32 x i16>* - %load.a = load <32 x i16>, <32 x i16>* %ptr.a, align 2 - %load.b = load <32 x i16>, <32 x i16>* %ptr.b, align 2 - %cmp = icmp uge <32 x i16> %load.a, %load.b - %sel = select <32 x i1> %cmp, <32 x i16> %load.b, <32 x i16> %load.a - store <32 x i16> %sel, <32 x i16>* %ptr.a, align 2 - %index.next = add i64 %index, 16 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminuw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ugt <32 x i16> %a, %b + %sel = select <32 x i1> %cmp, <32 x i16> %b, <32 x i16> %a + ret <32 x i16> %sel +} + +define <32 x i16> @test144(<32 x i16> %a, <32 x i16> %b) { +; SSE2-LABEL: test144: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: movdqa %xmm1, %xmm10 +; SSE2-NEXT: movdqa %xmm7, %xmm3 +; SSE2-NEXT: psubusw %xmm8, %xmm3 +; SSE2-NEXT: pxor %xmm12, %xmm12 +; SSE2-NEXT: pcmpeqw %xmm12, %xmm3 +; SSE2-NEXT: movdqa %xmm6, %xmm2 +; SSE2-NEXT: psubusw %xmm9, %xmm2 +; SSE2-NEXT: pcmpeqw %xmm12, %xmm2 +; SSE2-NEXT: movdqa %xmm5, %xmm1 +; SSE2-NEXT: psubusw %xmm10, %xmm1 +; SSE2-NEXT: pcmpeqw %xmm12, %xmm1 +; SSE2-NEXT: movdqa %xmm4, %xmm11 +; SSE2-NEXT: psubusw %xmm0, %xmm11 +; SSE2-NEXT: pcmpeqw %xmm12, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm11 +; SSE2-NEXT: por %xmm4, %xmm11 +; SSE2-NEXT: pand %xmm1, %xmm5 +; SSE2-NEXT: pandn %xmm10, %xmm1 +; SSE2-NEXT: por %xmm5, %xmm1 +; SSE2-NEXT: pand %xmm2, %xmm6 +; SSE2-NEXT: pandn %xmm9, %xmm2 +; SSE2-NEXT: por %xmm6, %xmm2 +; SSE2-NEXT: pand %xmm3, %xmm7 +; SSE2-NEXT: pandn %xmm8, %xmm3 +; SSE2-NEXT: por %xmm7, %xmm3 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test144: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminuw %xmm4, %xmm0 +; SSE4-NEXT: pminuw %xmm5, %xmm1 +; SSE4-NEXT: pminuw %xmm6, %xmm2 +; SSE4-NEXT: pminuw %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test144: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminuw %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminuw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminuw %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminuw %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test144: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminuw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminuw %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512BW-LABEL: test144: -; AVX512BW: vpminuw -} - -define void @test145(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <16 x i32>* - %ptr.b = bitcast i32* %gep.b to <16 x i32>* - %load.a = load <16 x i32>, <16 x i32>* %ptr.a, align 2 - %load.b = load <16 x i32>, <16 x i32>* %ptr.b, align 2 - %cmp = icmp slt <16 x i32> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i32> %load.b, <16 x i32> %load.a - store <16 x i32> %sel, <16 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminuw %zmm1, %zmm0, %zmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp uge <32 x i16> %a, %b + %sel = select <32 x i1> %cmp, <32 x i16> %b, <32 x i16> %a + ret <32 x i16> %sel +} + +define <16 x i32> @test145(<16 x i32> %a, <16 x i32> %b) { +; SSE2-LABEL: test145: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: movdqa %xmm7, %xmm3 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm3 +; SSE2-NEXT: movdqa %xmm6, %xmm9 +; SSE2-NEXT: pcmpgtd %xmm2, %xmm9 +; SSE2-NEXT: movdqa %xmm5, %xmm10 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm10 +; SSE2-NEXT: movdqa %xmm4, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm11 +; SSE2-NEXT: por %xmm4, %xmm11 +; SSE2-NEXT: pand %xmm10, %xmm5 +; SSE2-NEXT: pandn %xmm1, %xmm10 +; SSE2-NEXT: por %xmm5, %xmm10 +; SSE2-NEXT: pand %xmm9, %xmm6 +; SSE2-NEXT: pandn %xmm2, %xmm9 +; SSE2-NEXT: por %xmm6, %xmm9 +; SSE2-NEXT: pand %xmm3, %xmm7 +; SSE2-NEXT: pandn %xmm8, %xmm3 +; SSE2-NEXT: por %xmm7, %xmm3 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test145: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsd %xmm4, %xmm0 +; SSE4-NEXT: pmaxsd %xmm5, %xmm1 +; SSE4-NEXT: pmaxsd %xmm6, %xmm2 +; SSE4-NEXT: pmaxsd %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test145: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxsd %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxsd %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxsd %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxsd %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test145: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsd %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxsd %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test145: -; AVX512F: vpmaxsd -} - -define void @test146(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <16 x i32>* - %ptr.b = bitcast i32* %gep.b to <16 x i32>* - %load.a = load <16 x i32>, <16 x i32>* %ptr.a, align 2 - %load.b = load <16 x i32>, <16 x i32>* %ptr.b, align 2 - %cmp = icmp sle <16 x i32> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i32> %load.b, <16 x i32> %load.a - store <16 x i32> %sel, <16 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsd %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp slt <16 x i32> %a, %b + %sel = select <16 x i1> %cmp, <16 x i32> %b, <16 x i32> %a + ret <16 x i32> %sel +} + +define <16 x i32> @test146(<16 x i32> %a, <16 x i32> %b) { +; SSE2-LABEL: test146: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm2, %xmm8 +; SSE2-NEXT: movdqa %xmm3, %xmm12 +; SSE2-NEXT: pcmpgtd %xmm7, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm13, %xmm13 +; SSE2-NEXT: movdqa %xmm12, %xmm9 +; SSE2-NEXT: pxor %xmm13, %xmm9 +; SSE2-NEXT: movdqa %xmm8, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm14 +; SSE2-NEXT: movdqa %xmm14, %xmm2 +; SSE2-NEXT: pxor %xmm13, %xmm2 +; SSE2-NEXT: movdqa %xmm1, %xmm15 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm15 +; SSE2-NEXT: movdqa %xmm15, %xmm10 +; SSE2-NEXT: pxor %xmm13, %xmm10 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm11 +; SSE2-NEXT: pxor %xmm11, %xmm13 +; SSE2-NEXT: pandn %xmm4, %xmm11 +; SSE2-NEXT: pandn %xmm0, %xmm13 +; SSE2-NEXT: por %xmm13, %xmm11 +; SSE2-NEXT: pandn %xmm5, %xmm15 +; SSE2-NEXT: pandn %xmm1, %xmm10 +; SSE2-NEXT: por %xmm15, %xmm10 +; SSE2-NEXT: pandn %xmm6, %xmm14 +; SSE2-NEXT: pandn %xmm8, %xmm2 +; SSE2-NEXT: por %xmm14, %xmm2 +; SSE2-NEXT: pandn %xmm7, %xmm12 +; SSE2-NEXT: pandn %xmm3, %xmm9 +; SSE2-NEXT: por %xmm12, %xmm9 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test146: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxsd %xmm4, %xmm0 +; SSE4-NEXT: pmaxsd %xmm5, %xmm1 +; SSE4-NEXT: pmaxsd %xmm6, %xmm2 +; SSE4-NEXT: pmaxsd %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test146: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxsd %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxsd %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxsd %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxsd %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test146: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxsd %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxsd %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test146: -; AVX512F: vpmaxsd -} - -define void @test147(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <16 x i32>* - %ptr.b = bitcast i32* %gep.b to <16 x i32>* - %load.a = load <16 x i32>, <16 x i32>* %ptr.a, align 2 - %load.b = load <16 x i32>, <16 x i32>* %ptr.b, align 2 - %cmp = icmp sgt <16 x i32> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i32> %load.b, <16 x i32> %load.a - store <16 x i32> %sel, <16 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsd %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sle <16 x i32> %a, %b + %sel = select <16 x i1> %cmp, <16 x i32> %b, <16 x i32> %a + ret <16 x i32> %sel +} + +define <16 x i32> @test147(<16 x i32> %a, <16 x i32> %b) { +; SSE2-LABEL: test147: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: pcmpgtd %xmm7, %xmm3 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm9 +; SSE2-NEXT: movdqa %xmm1, %xmm10 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm10 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm11 +; SSE2-NEXT: por %xmm4, %xmm11 +; SSE2-NEXT: pand %xmm10, %xmm5 +; SSE2-NEXT: pandn %xmm1, %xmm10 +; SSE2-NEXT: por %xmm5, %xmm10 +; SSE2-NEXT: pand %xmm9, %xmm6 +; SSE2-NEXT: pandn %xmm2, %xmm9 +; SSE2-NEXT: por %xmm6, %xmm9 +; SSE2-NEXT: pand %xmm3, %xmm7 +; SSE2-NEXT: pandn %xmm8, %xmm3 +; SSE2-NEXT: por %xmm7, %xmm3 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test147: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsd %xmm4, %xmm0 +; SSE4-NEXT: pminsd %xmm5, %xmm1 +; SSE4-NEXT: pminsd %xmm6, %xmm2 +; SSE4-NEXT: pminsd %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test147: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminsd %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminsd %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminsd %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminsd %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test147: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsd %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminsd %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test147: -; AVX512F: vpminsd -} - -define void @test148(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <16 x i32>* - %ptr.b = bitcast i32* %gep.b to <16 x i32>* - %load.a = load <16 x i32>, <16 x i32>* %ptr.a, align 2 - %load.b = load <16 x i32>, <16 x i32>* %ptr.b, align 2 - %cmp = icmp sge <16 x i32> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i32> %load.b, <16 x i32> %load.a - store <16 x i32> %sel, <16 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsd %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sgt <16 x i32> %a, %b + %sel = select <16 x i1> %cmp, <16 x i32> %b, <16 x i32> %a + ret <16 x i32> %sel +} + +define <16 x i32> @test148(<16 x i32> %a, <16 x i32> %b) { +; SSE2-LABEL: test148: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm2, %xmm8 +; SSE2-NEXT: movdqa %xmm0, %xmm10 +; SSE2-NEXT: movdqa %xmm7, %xmm12 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE2-NEXT: movdqa %xmm12, %xmm9 +; SSE2-NEXT: pxor %xmm0, %xmm9 +; SSE2-NEXT: movdqa %xmm6, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm13 +; SSE2-NEXT: movdqa %xmm13, %xmm2 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm5, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm14 +; SSE2-NEXT: movdqa %xmm14, %xmm11 +; SSE2-NEXT: pxor %xmm0, %xmm11 +; SSE2-NEXT: movdqa %xmm4, %xmm15 +; SSE2-NEXT: pcmpgtd %xmm10, %xmm15 +; SSE2-NEXT: pxor %xmm15, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm15 +; SSE2-NEXT: pandn %xmm10, %xmm0 +; SSE2-NEXT: por %xmm15, %xmm0 +; SSE2-NEXT: pandn %xmm5, %xmm14 +; SSE2-NEXT: pandn %xmm1, %xmm11 +; SSE2-NEXT: por %xmm14, %xmm11 +; SSE2-NEXT: pandn %xmm6, %xmm13 +; SSE2-NEXT: pandn %xmm8, %xmm2 +; SSE2-NEXT: por %xmm13, %xmm2 +; SSE2-NEXT: pandn %xmm7, %xmm12 +; SSE2-NEXT: pandn %xmm3, %xmm9 +; SSE2-NEXT: por %xmm12, %xmm9 +; SSE2-NEXT: movdqa %xmm11, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test148: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminsd %xmm4, %xmm0 +; SSE4-NEXT: pminsd %xmm5, %xmm1 +; SSE4-NEXT: pminsd %xmm6, %xmm2 +; SSE4-NEXT: pminsd %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test148: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminsd %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminsd %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminsd %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminsd %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test148: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminsd %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminsd %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test148: -; AVX512F: vpminsd -} - -define void @test149(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <16 x i32>* - %ptr.b = bitcast i32* %gep.b to <16 x i32>* - %load.a = load <16 x i32>, <16 x i32>* %ptr.a, align 2 - %load.b = load <16 x i32>, <16 x i32>* %ptr.b, align 2 - %cmp = icmp ult <16 x i32> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i32> %load.b, <16 x i32> %load.a - store <16 x i32> %sel, <16 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsd %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sge <16 x i32> %a, %b + %sel = select <16 x i1> %cmp, <16 x i32> %b, <16 x i32> %a + ret <16 x i32> %sel +} + +define <16 x i32> @test149(<16 x i32> %a, <16 x i32> %b) { +; SSE2-LABEL: test149: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm9 +; SSE2-NEXT: pxor %xmm0, %xmm9 +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: pxor %xmm0, %xmm8 +; SSE2-NEXT: pcmpgtd %xmm9, %xmm8 +; SSE2-NEXT: movdqa %xmm2, %xmm10 +; SSE2-NEXT: pxor %xmm0, %xmm10 +; SSE2-NEXT: movdqa %xmm6, %xmm9 +; SSE2-NEXT: pxor %xmm0, %xmm9 +; SSE2-NEXT: pcmpgtd %xmm10, %xmm9 +; SSE2-NEXT: movdqa %xmm1, %xmm12 +; SSE2-NEXT: pxor %xmm0, %xmm12 +; SSE2-NEXT: movdqa %xmm5, %xmm10 +; SSE2-NEXT: pxor %xmm0, %xmm10 +; SSE2-NEXT: pcmpgtd %xmm12, %xmm10 +; SSE2-NEXT: movdqa %xmm11, %xmm12 +; SSE2-NEXT: pxor %xmm0, %xmm12 +; SSE2-NEXT: pxor %xmm4, %xmm0 +; SSE2-NEXT: pcmpgtd %xmm12, %xmm0 +; SSE2-NEXT: pand %xmm0, %xmm4 +; SSE2-NEXT: pandn %xmm11, %xmm0 +; SSE2-NEXT: por %xmm4, %xmm0 +; SSE2-NEXT: pand %xmm10, %xmm5 +; SSE2-NEXT: pandn %xmm1, %xmm10 +; SSE2-NEXT: por %xmm5, %xmm10 +; SSE2-NEXT: pand %xmm9, %xmm6 +; SSE2-NEXT: pandn %xmm2, %xmm9 +; SSE2-NEXT: por %xmm6, %xmm9 +; SSE2-NEXT: pand %xmm8, %xmm7 +; SSE2-NEXT: pandn %xmm3, %xmm8 +; SSE2-NEXT: por %xmm7, %xmm8 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test149: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxud %xmm4, %xmm0 +; SSE4-NEXT: pmaxud %xmm5, %xmm1 +; SSE4-NEXT: pmaxud %xmm6, %xmm2 +; SSE4-NEXT: pmaxud %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test149: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxud %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxud %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxud %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxud %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test149: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxud %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxud %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test149: -; AVX512F: vpmaxud -} - -define void @test150(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <16 x i32>* - %ptr.b = bitcast i32* %gep.b to <16 x i32>* - %load.a = load <16 x i32>, <16 x i32>* %ptr.a, align 2 - %load.b = load <16 x i32>, <16 x i32>* %ptr.b, align 2 - %cmp = icmp ule <16 x i32> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i32> %load.b, <16 x i32> %load.a - store <16 x i32> %sel, <16 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxud %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ult <16 x i32> %a, %b + %sel = select <16 x i1> %cmp, <16 x i32> %b, <16 x i32> %a + ret <16 x i32> %sel +} + +define <16 x i32> @test150(<16 x i32> %a, <16 x i32> %b) { +; SSE2-LABEL: test150: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm10 +; SSE2-NEXT: movdqa {{.*#+}} xmm14 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm7, %xmm0 +; SSE2-NEXT: pxor %xmm14, %xmm0 +; SSE2-NEXT: movdqa %xmm3, %xmm12 +; SSE2-NEXT: pxor %xmm14, %xmm12 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE2-NEXT: movdqa %xmm12, %xmm8 +; SSE2-NEXT: pxor %xmm0, %xmm8 +; SSE2-NEXT: movdqa %xmm6, %xmm9 +; SSE2-NEXT: pxor %xmm14, %xmm9 +; SSE2-NEXT: movdqa %xmm2, %xmm13 +; SSE2-NEXT: pxor %xmm14, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm9, %xmm13 +; SSE2-NEXT: movdqa %xmm13, %xmm9 +; SSE2-NEXT: pxor %xmm0, %xmm9 +; SSE2-NEXT: movdqa %xmm5, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm11 +; SSE2-NEXT: movdqa %xmm1, %xmm15 +; SSE2-NEXT: pxor %xmm14, %xmm15 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm15 +; SSE2-NEXT: movdqa %xmm4, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: movdqa %xmm15, %xmm11 +; SSE2-NEXT: pxor %xmm0, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm14 +; SSE2-NEXT: pandn %xmm10, %xmm0 +; SSE2-NEXT: por %xmm14, %xmm0 +; SSE2-NEXT: pandn %xmm5, %xmm15 +; SSE2-NEXT: pandn %xmm1, %xmm11 +; SSE2-NEXT: por %xmm15, %xmm11 +; SSE2-NEXT: pandn %xmm6, %xmm13 +; SSE2-NEXT: pandn %xmm2, %xmm9 +; SSE2-NEXT: por %xmm13, %xmm9 +; SSE2-NEXT: pandn %xmm7, %xmm12 +; SSE2-NEXT: pandn %xmm3, %xmm8 +; SSE2-NEXT: por %xmm12, %xmm8 +; SSE2-NEXT: movdqa %xmm11, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test150: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pmaxud %xmm4, %xmm0 +; SSE4-NEXT: pmaxud %xmm5, %xmm1 +; SSE4-NEXT: pmaxud %xmm6, %xmm2 +; SSE4-NEXT: pmaxud %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test150: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpmaxud %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpmaxud %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpmaxud %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpmaxud %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test150: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpmaxud %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpmaxud %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test150: -; AVX512F: vpmaxud -} - -define void @test151(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <16 x i32>* - %ptr.b = bitcast i32* %gep.b to <16 x i32>* - %load.a = load <16 x i32>, <16 x i32>* %ptr.a, align 2 - %load.b = load <16 x i32>, <16 x i32>* %ptr.b, align 2 - %cmp = icmp ugt <16 x i32> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i32> %load.b, <16 x i32> %load.a - store <16 x i32> %sel, <16 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxud %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ule <16 x i32> %a, %b + %sel = select <16 x i1> %cmp, <16 x i32> %b, <16 x i32> %a + ret <16 x i32> %sel +} + +define <16 x i32> @test151(<16 x i32> %a, <16 x i32> %b) { +; SSE2-LABEL: test151: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm7, %xmm9 +; SSE2-NEXT: pxor %xmm0, %xmm9 +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: pxor %xmm0, %xmm8 +; SSE2-NEXT: pcmpgtd %xmm9, %xmm8 +; SSE2-NEXT: movdqa %xmm6, %xmm10 +; SSE2-NEXT: pxor %xmm0, %xmm10 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: pxor %xmm0, %xmm9 +; SSE2-NEXT: pcmpgtd %xmm10, %xmm9 +; SSE2-NEXT: movdqa %xmm5, %xmm12 +; SSE2-NEXT: pxor %xmm0, %xmm12 +; SSE2-NEXT: movdqa %xmm1, %xmm10 +; SSE2-NEXT: pxor %xmm0, %xmm10 +; SSE2-NEXT: pcmpgtd %xmm12, %xmm10 +; SSE2-NEXT: movdqa %xmm4, %xmm12 +; SSE2-NEXT: pxor %xmm0, %xmm12 +; SSE2-NEXT: pxor %xmm11, %xmm0 +; SSE2-NEXT: pcmpgtd %xmm12, %xmm0 +; SSE2-NEXT: pand %xmm0, %xmm4 +; SSE2-NEXT: pandn %xmm11, %xmm0 +; SSE2-NEXT: por %xmm4, %xmm0 +; SSE2-NEXT: pand %xmm10, %xmm5 +; SSE2-NEXT: pandn %xmm1, %xmm10 +; SSE2-NEXT: por %xmm5, %xmm10 +; SSE2-NEXT: pand %xmm9, %xmm6 +; SSE2-NEXT: pandn %xmm2, %xmm9 +; SSE2-NEXT: por %xmm6, %xmm9 +; SSE2-NEXT: pand %xmm8, %xmm7 +; SSE2-NEXT: pandn %xmm3, %xmm8 +; SSE2-NEXT: por %xmm7, %xmm8 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test151: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminud %xmm4, %xmm0 +; SSE4-NEXT: pminud %xmm5, %xmm1 +; SSE4-NEXT: pminud %xmm6, %xmm2 +; SSE4-NEXT: pminud %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test151: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminud %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminud %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminud %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminud %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test151: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminud %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminud %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test151: -; AVX512F: vpminud -} - -define void @test152(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <16 x i32>* - %ptr.b = bitcast i32* %gep.b to <16 x i32>* - %load.a = load <16 x i32>, <16 x i32>* %ptr.a, align 2 - %load.b = load <16 x i32>, <16 x i32>* %ptr.b, align 2 - %cmp = icmp uge <16 x i32> %load.a, %load.b - %sel = select <16 x i1> %cmp, <16 x i32> %load.b, <16 x i32> %load.a - store <16 x i32> %sel, <16 x i32>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminud %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ugt <16 x i32> %a, %b + %sel = select <16 x i1> %cmp, <16 x i32> %b, <16 x i32> %a + ret <16 x i32> %sel +} + +define <16 x i32> @test152(<16 x i32> %a, <16 x i32> %b) { +; SSE2-LABEL: test152: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm0, %xmm10 +; SSE2-NEXT: movdqa {{.*#+}} xmm14 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm0 +; SSE2-NEXT: pxor %xmm14, %xmm0 +; SSE2-NEXT: movdqa %xmm7, %xmm12 +; SSE2-NEXT: pxor %xmm14, %xmm12 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE2-NEXT: movdqa %xmm12, %xmm8 +; SSE2-NEXT: pxor %xmm0, %xmm8 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: pxor %xmm14, %xmm9 +; SSE2-NEXT: movdqa %xmm6, %xmm13 +; SSE2-NEXT: pxor %xmm14, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm9, %xmm13 +; SSE2-NEXT: movdqa %xmm13, %xmm9 +; SSE2-NEXT: pxor %xmm0, %xmm9 +; SSE2-NEXT: movdqa %xmm1, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm11 +; SSE2-NEXT: movdqa %xmm5, %xmm15 +; SSE2-NEXT: pxor %xmm14, %xmm15 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm15 +; SSE2-NEXT: movdqa %xmm10, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm11 +; SSE2-NEXT: pxor %xmm4, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: movdqa %xmm15, %xmm11 +; SSE2-NEXT: pxor %xmm0, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm0 +; SSE2-NEXT: pandn %xmm4, %xmm14 +; SSE2-NEXT: pandn %xmm10, %xmm0 +; SSE2-NEXT: por %xmm14, %xmm0 +; SSE2-NEXT: pandn %xmm5, %xmm15 +; SSE2-NEXT: pandn %xmm1, %xmm11 +; SSE2-NEXT: por %xmm15, %xmm11 +; SSE2-NEXT: pandn %xmm6, %xmm13 +; SSE2-NEXT: pandn %xmm2, %xmm9 +; SSE2-NEXT: por %xmm13, %xmm9 +; SSE2-NEXT: pandn %xmm7, %xmm12 +; SSE2-NEXT: pandn %xmm3, %xmm8 +; SSE2-NEXT: por %xmm12, %xmm8 +; SSE2-NEXT: movdqa %xmm11, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test152: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: pminud %xmm4, %xmm0 +; SSE4-NEXT: pminud %xmm5, %xmm1 +; SSE4-NEXT: pminud %xmm6, %xmm2 +; SSE4-NEXT: pminud %xmm7, %xmm3 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test152: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vpminud %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpminud %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vpminud %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpminud %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test152: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpminud %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpminud %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test152: -; AVX512F: vpminud +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminud %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp uge <16 x i32> %a, %b + %sel = select <16 x i1> %cmp, <16 x i32> %b, <16 x i32> %a + ret <16 x i32> %sel } ; ----------------------- -define void @test153(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i64>* - %ptr.b = bitcast i32* %gep.b to <8 x i64>* - %load.a = load <8 x i64>, <8 x i64>* %ptr.a, align 2 - %load.b = load <8 x i64>, <8 x i64>* %ptr.b, align 2 - %cmp = icmp slt <8 x i64> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i64> %load.b, <8 x i64> %load.a - store <8 x i64> %sel, <8 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +define <8 x i64> @test153(<8 x i64> %a, <8 x i64> %b) { +; SSE2-LABEL: test153: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm11 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: pxor %xmm11, %xmm8 +; SSE2-NEXT: movdqa %xmm7, %xmm9 +; SSE2-NEXT: pxor %xmm11, %xmm9 +; SSE2-NEXT: movdqa %xmm9, %xmm10 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm10[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm8, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3] +; SSE2-NEXT: pand %xmm12, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm10[1,1,3,3] +; SSE2-NEXT: por %xmm9, %xmm8 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: pxor %xmm11, %xmm9 +; SSE2-NEXT: movdqa %xmm6, %xmm10 +; SSE2-NEXT: pxor %xmm11, %xmm10 +; SSE2-NEXT: movdqa %xmm10, %xmm12 +; SSE2-NEXT: pcmpgtd %xmm9, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm9, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3] +; SSE2-NEXT: pand %xmm13, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm12[1,1,3,3] +; SSE2-NEXT: por %xmm10, %xmm9 +; SSE2-NEXT: movdqa %xmm1, %xmm10 +; SSE2-NEXT: pxor %xmm11, %xmm10 +; SSE2-NEXT: movdqa %xmm5, %xmm12 +; SSE2-NEXT: pxor %xmm11, %xmm12 +; SSE2-NEXT: movdqa %xmm12, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm10, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm10, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3] +; SSE2-NEXT: pand %xmm14, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm13[1,1,3,3] +; SSE2-NEXT: por %xmm12, %xmm10 +; SSE2-NEXT: movdqa %xmm0, %xmm12 +; SSE2-NEXT: pxor %xmm11, %xmm12 +; SSE2-NEXT: pxor %xmm4, %xmm11 +; SSE2-NEXT: movdqa %xmm11, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm12, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm12, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[1,1,3,3] +; SSE2-NEXT: pand %xmm14, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[1,1,3,3] +; SSE2-NEXT: por %xmm12, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm11 +; SSE2-NEXT: por %xmm4, %xmm11 +; SSE2-NEXT: pand %xmm10, %xmm5 +; SSE2-NEXT: pandn %xmm1, %xmm10 +; SSE2-NEXT: por %xmm5, %xmm10 +; SSE2-NEXT: pand %xmm9, %xmm6 +; SSE2-NEXT: pandn %xmm2, %xmm9 +; SSE2-NEXT: por %xmm6, %xmm9 +; SSE2-NEXT: pand %xmm8, %xmm7 +; SSE2-NEXT: pandn %xmm3, %xmm8 +; SSE2-NEXT: por %xmm7, %xmm8 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test153: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm8 +; SSE4-NEXT: movdqa %xmm7, %xmm9 +; SSE4-NEXT: pcmpgtq %xmm3, %xmm9 +; SSE4-NEXT: movdqa %xmm6, %xmm10 +; SSE4-NEXT: pcmpgtq %xmm2, %xmm10 +; SSE4-NEXT: movdqa %xmm5, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm1, %xmm11 +; SSE4-NEXT: movdqa %xmm4, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm8, %xmm0 +; SSE4-NEXT: blendvpd %xmm4, %xmm8 +; SSE4-NEXT: movdqa %xmm11, %xmm0 +; SSE4-NEXT: blendvpd %xmm5, %xmm1 +; SSE4-NEXT: movdqa %xmm10, %xmm0 +; SSE4-NEXT: blendvpd %xmm6, %xmm2 +; SSE4-NEXT: movdqa %xmm9, %xmm0 +; SSE4-NEXT: blendvpd %xmm7, %xmm3 +; SSE4-NEXT: movapd %xmm8, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test153: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm1, %xmm3, %xmm5 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm5, %xmm6, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm0, %xmm2, %xmm6 +; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm6, %ymm5 +; AVX1-NEXT: vblendvpd %ymm5, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vblendvpd %ymm4, %ymm3, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test153: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %ymm1, %ymm3, %ymm4 +; AVX2-NEXT: vpcmpgtq %ymm0, %ymm2, %ymm5 +; AVX2-NEXT: vblendvpd %ymm5, %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vblendvpd %ymm4, %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test153: -; AVX512F: vpmaxsq -} - -define void @test154(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i64>* - %ptr.b = bitcast i32* %gep.b to <8 x i64>* - %load.a = load <8 x i64>, <8 x i64>* %ptr.a, align 2 - %load.b = load <8 x i64>, <8 x i64>* %ptr.b, align 2 - %cmp = icmp sle <8 x i64> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i64> %load.b, <8 x i64> %load.a - store <8 x i64> %sel, <8 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp slt <8 x i64> %a, %b + %sel = select <8 x i1> %cmp, <8 x i64> %b, <8 x i64> %a + ret <8 x i64> %sel +} + +define <8 x i64> @test154(<8 x i64> %a, <8 x i64> %b) { +; SSE2-LABEL: test154: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: movdqa %xmm8, -{{[0-9]+}}(%rsp) # 16-byte Spill +; SSE2-NEXT: movdqa %xmm3, %xmm7 +; SSE2-NEXT: movdqa %xmm2, %xmm3 +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm0, %xmm9 +; SSE2-NEXT: movdqa {{.*#+}} xmm10 = [2147483648,0,2147483648,0] +; SSE2-NEXT: pxor %xmm10, %xmm8 +; SSE2-NEXT: movdqa %xmm7, %xmm0 +; SSE2-NEXT: pxor %xmm10, %xmm0 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm8, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE2-NEXT: pand %xmm12, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: movdqa %xmm12, %xmm8 +; SSE2-NEXT: pxor %xmm1, %xmm8 +; SSE2-NEXT: movdqa %xmm6, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm3, %xmm13 +; SSE2-NEXT: pxor %xmm10, %xmm13 +; SSE2-NEXT: movdqa %xmm13, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm11, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[1,1,3,3] +; SSE2-NEXT: pand %xmm15, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm14[1,1,3,3] +; SSE2-NEXT: por %xmm11, %xmm13 +; SSE2-NEXT: movdqa %xmm5, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm2, %xmm14 +; SSE2-NEXT: pxor %xmm10, %xmm14 +; SSE2-NEXT: movdqa %xmm14, %xmm15 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm15 +; SSE2-NEXT: pcmpeqd %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm15[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm14[1,1,3,3] +; SSE2-NEXT: pand %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm15[1,1,3,3] +; SSE2-NEXT: por %xmm14, %xmm15 +; SSE2-NEXT: movdqa %xmm4, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: pxor %xmm9, %xmm10 +; SSE2-NEXT: movdqa %xmm10, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: pcmpeqd %xmm11, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm14[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm13, %xmm10 +; SSE2-NEXT: pxor %xmm1, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm14[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm14 +; SSE2-NEXT: movdqa %xmm15, %xmm11 +; SSE2-NEXT: pxor %xmm1, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm1 +; SSE2-NEXT: pandn %xmm4, %xmm14 +; SSE2-NEXT: pandn %xmm9, %xmm1 +; SSE2-NEXT: por %xmm14, %xmm1 +; SSE2-NEXT: pandn %xmm5, %xmm15 +; SSE2-NEXT: pandn %xmm2, %xmm11 +; SSE2-NEXT: por %xmm15, %xmm11 +; SSE2-NEXT: pandn %xmm6, %xmm13 +; SSE2-NEXT: pandn %xmm3, %xmm10 +; SSE2-NEXT: por %xmm13, %xmm10 +; SSE2-NEXT: pandn -{{[0-9]+}}(%rsp), %xmm12 # 16-byte Folded Reload +; SSE2-NEXT: pandn %xmm7, %xmm8 +; SSE2-NEXT: por %xmm12, %xmm8 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: movdqa %xmm11, %xmm1 +; SSE2-NEXT: movdqa %xmm10, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test154: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm8 +; SSE4-NEXT: movdqa %xmm3, %xmm9 +; SSE4-NEXT: pcmpgtq %xmm7, %xmm9 +; SSE4-NEXT: pcmpeqd %xmm12, %xmm12 +; SSE4-NEXT: pxor %xmm12, %xmm9 +; SSE4-NEXT: movdqa %xmm2, %xmm10 +; SSE4-NEXT: pcmpgtq %xmm6, %xmm10 +; SSE4-NEXT: pxor %xmm12, %xmm10 +; SSE4-NEXT: movdqa %xmm1, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm5, %xmm11 +; SSE4-NEXT: pxor %xmm12, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm4, %xmm0 +; SSE4-NEXT: pxor %xmm12, %xmm0 +; SSE4-NEXT: blendvpd %xmm4, %xmm8 +; SSE4-NEXT: movdqa %xmm11, %xmm0 +; SSE4-NEXT: blendvpd %xmm5, %xmm1 +; SSE4-NEXT: movdqa %xmm10, %xmm0 +; SSE4-NEXT: blendvpd %xmm6, %xmm2 +; SSE4-NEXT: movdqa %xmm9, %xmm0 +; SSE4-NEXT: blendvpd %xmm7, %xmm3 +; SSE4-NEXT: movapd %xmm8, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test154: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpcmpeqd %xmm5, %xmm5, %xmm5 +; AVX1-NEXT: vpxor %xmm5, %xmm4, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm1, %xmm6 +; AVX1-NEXT: vpxor %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4 +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm7 +; AVX1-NEXT: vpcmpgtq %xmm6, %xmm7, %xmm6 +; AVX1-NEXT: vpxor %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm0, %xmm7 +; AVX1-NEXT: vpxor %xmm5, %xmm7, %xmm5 +; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5 +; AVX1-NEXT: vblendvpd %ymm5, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vblendvpd %ymm4, %ymm3, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test154: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %ymm3, %ymm1, %ymm4 +; AVX2-NEXT: vpcmpeqd %ymm5, %ymm5, %ymm5 +; AVX2-NEXT: vpxor %ymm5, %ymm4, %ymm4 +; AVX2-NEXT: vpcmpgtq %ymm2, %ymm0, %ymm6 +; AVX2-NEXT: vpxor %ymm5, %ymm6, %ymm5 +; AVX2-NEXT: vblendvpd %ymm5, %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vblendvpd %ymm4, %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test154: -; AVX512F: vpmaxsq -} - -define void @test155(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i64>* - %ptr.b = bitcast i32* %gep.b to <8 x i64>* - %load.a = load <8 x i64>, <8 x i64>* %ptr.a, align 2 - %load.b = load <8 x i64>, <8 x i64>* %ptr.b, align 2 - %cmp = icmp sgt <8 x i64> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i64> %load.b, <8 x i64> %load.a - store <8 x i64> %sel, <8 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sle <8 x i64> %a, %b + %sel = select <8 x i1> %cmp, <8 x i64> %b, <8 x i64> %a + ret <8 x i64> %sel +} + +define <8 x i64> @test155(<8 x i64> %a, <8 x i64> %b) { +; SSE2-LABEL: test155: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm11 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: pxor %xmm11, %xmm8 +; SSE2-NEXT: movdqa %xmm3, %xmm9 +; SSE2-NEXT: pxor %xmm11, %xmm9 +; SSE2-NEXT: movdqa %xmm9, %xmm10 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm10[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm8, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3] +; SSE2-NEXT: pand %xmm12, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm10[1,1,3,3] +; SSE2-NEXT: por %xmm9, %xmm8 +; SSE2-NEXT: movdqa %xmm6, %xmm9 +; SSE2-NEXT: pxor %xmm11, %xmm9 +; SSE2-NEXT: movdqa %xmm2, %xmm10 +; SSE2-NEXT: pxor %xmm11, %xmm10 +; SSE2-NEXT: movdqa %xmm10, %xmm12 +; SSE2-NEXT: pcmpgtd %xmm9, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm9, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3] +; SSE2-NEXT: pand %xmm13, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm12[1,1,3,3] +; SSE2-NEXT: por %xmm10, %xmm9 +; SSE2-NEXT: movdqa %xmm5, %xmm10 +; SSE2-NEXT: pxor %xmm11, %xmm10 +; SSE2-NEXT: movdqa %xmm1, %xmm12 +; SSE2-NEXT: pxor %xmm11, %xmm12 +; SSE2-NEXT: movdqa %xmm12, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm10, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm10, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3] +; SSE2-NEXT: pand %xmm14, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm13[1,1,3,3] +; SSE2-NEXT: por %xmm12, %xmm10 +; SSE2-NEXT: movdqa %xmm4, %xmm12 +; SSE2-NEXT: pxor %xmm11, %xmm12 +; SSE2-NEXT: pxor %xmm0, %xmm11 +; SSE2-NEXT: movdqa %xmm11, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm12, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm12, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[1,1,3,3] +; SSE2-NEXT: pand %xmm14, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[1,1,3,3] +; SSE2-NEXT: por %xmm12, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm11 +; SSE2-NEXT: por %xmm4, %xmm11 +; SSE2-NEXT: pand %xmm10, %xmm5 +; SSE2-NEXT: pandn %xmm1, %xmm10 +; SSE2-NEXT: por %xmm5, %xmm10 +; SSE2-NEXT: pand %xmm9, %xmm6 +; SSE2-NEXT: pandn %xmm2, %xmm9 +; SSE2-NEXT: por %xmm6, %xmm9 +; SSE2-NEXT: pand %xmm8, %xmm7 +; SSE2-NEXT: pandn %xmm3, %xmm8 +; SSE2-NEXT: por %xmm7, %xmm8 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test155: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm8 +; SSE4-NEXT: movdqa %xmm3, %xmm9 +; SSE4-NEXT: pcmpgtq %xmm7, %xmm9 +; SSE4-NEXT: movdqa %xmm2, %xmm10 +; SSE4-NEXT: pcmpgtq %xmm6, %xmm10 +; SSE4-NEXT: movdqa %xmm1, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm5, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm4, %xmm0 +; SSE4-NEXT: blendvpd %xmm4, %xmm8 +; SSE4-NEXT: movdqa %xmm11, %xmm0 +; SSE4-NEXT: blendvpd %xmm5, %xmm1 +; SSE4-NEXT: movdqa %xmm10, %xmm0 +; SSE4-NEXT: blendvpd %xmm6, %xmm2 +; SSE4-NEXT: movdqa %xmm9, %xmm0 +; SSE4-NEXT: blendvpd %xmm7, %xmm3 +; SSE4-NEXT: movapd %xmm8, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test155: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm1, %xmm5 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm4 +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm5 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm5, %xmm6, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm0, %xmm6 +; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm6, %ymm5 +; AVX1-NEXT: vblendvpd %ymm5, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vblendvpd %ymm4, %ymm3, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test155: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %ymm3, %ymm1, %ymm4 +; AVX2-NEXT: vpcmpgtq %ymm2, %ymm0, %ymm5 +; AVX2-NEXT: vblendvpd %ymm5, %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vblendvpd %ymm4, %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test155: -; AVX512F: vpminsq -} - -define void @test156(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i64>* - %ptr.b = bitcast i32* %gep.b to <8 x i64>* - %load.a = load <8 x i64>, <8 x i64>* %ptr.a, align 2 - %load.b = load <8 x i64>, <8 x i64>* %ptr.b, align 2 - %cmp = icmp sge <8 x i64> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i64> %load.b, <8 x i64> %load.a - store <8 x i64> %sel, <8 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sgt <8 x i64> %a, %b + %sel = select <8 x i1> %cmp, <8 x i64> %b, <8 x i64> %a + ret <8 x i64> %sel +} + +define <8 x i64> @test156(<8 x i64> %a, <8 x i64> %b) { +; SSE2-LABEL: test156: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm7, %xmm11 +; SSE2-NEXT: movdqa %xmm11, -{{[0-9]+}}(%rsp) # 16-byte Spill +; SSE2-NEXT: movdqa %xmm3, %xmm7 +; SSE2-NEXT: movdqa %xmm2, %xmm3 +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm0, %xmm9 +; SSE2-NEXT: movdqa {{.*#+}} xmm10 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: pxor %xmm10, %xmm8 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: pxor %xmm10, %xmm0 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm8, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE2-NEXT: pand %xmm12, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: movdqa %xmm12, %xmm8 +; SSE2-NEXT: pxor %xmm1, %xmm8 +; SSE2-NEXT: movdqa %xmm3, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm6, %xmm13 +; SSE2-NEXT: pxor %xmm10, %xmm13 +; SSE2-NEXT: movdqa %xmm13, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm11, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[1,1,3,3] +; SSE2-NEXT: pand %xmm15, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm14[1,1,3,3] +; SSE2-NEXT: por %xmm11, %xmm13 +; SSE2-NEXT: movdqa %xmm2, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm5, %xmm14 +; SSE2-NEXT: pxor %xmm10, %xmm14 +; SSE2-NEXT: movdqa %xmm14, %xmm15 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm15 +; SSE2-NEXT: pcmpeqd %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm15[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm14[1,1,3,3] +; SSE2-NEXT: pand %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm15[1,1,3,3] +; SSE2-NEXT: por %xmm14, %xmm15 +; SSE2-NEXT: movdqa %xmm9, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: pxor %xmm4, %xmm10 +; SSE2-NEXT: movdqa %xmm10, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: pcmpeqd %xmm11, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm14[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm13, %xmm10 +; SSE2-NEXT: pxor %xmm1, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm14[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm14 +; SSE2-NEXT: movdqa %xmm15, %xmm11 +; SSE2-NEXT: pxor %xmm1, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm1 +; SSE2-NEXT: pandn %xmm4, %xmm14 +; SSE2-NEXT: pandn %xmm9, %xmm1 +; SSE2-NEXT: por %xmm14, %xmm1 +; SSE2-NEXT: pandn %xmm5, %xmm15 +; SSE2-NEXT: pandn %xmm2, %xmm11 +; SSE2-NEXT: por %xmm15, %xmm11 +; SSE2-NEXT: pandn %xmm6, %xmm13 +; SSE2-NEXT: pandn %xmm3, %xmm10 +; SSE2-NEXT: por %xmm13, %xmm10 +; SSE2-NEXT: pandn -{{[0-9]+}}(%rsp), %xmm12 # 16-byte Folded Reload +; SSE2-NEXT: pandn %xmm7, %xmm8 +; SSE2-NEXT: por %xmm12, %xmm8 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: movdqa %xmm11, %xmm1 +; SSE2-NEXT: movdqa %xmm10, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test156: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm8 +; SSE4-NEXT: movdqa %xmm7, %xmm9 +; SSE4-NEXT: pcmpgtq %xmm3, %xmm9 +; SSE4-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE4-NEXT: pxor %xmm0, %xmm9 +; SSE4-NEXT: movdqa %xmm6, %xmm10 +; SSE4-NEXT: pcmpgtq %xmm2, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: movdqa %xmm5, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm1, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: movdqa %xmm4, %xmm12 +; SSE4-NEXT: pcmpgtq %xmm8, %xmm12 +; SSE4-NEXT: pxor %xmm12, %xmm0 +; SSE4-NEXT: blendvpd %xmm4, %xmm8 +; SSE4-NEXT: movdqa %xmm11, %xmm0 +; SSE4-NEXT: blendvpd %xmm5, %xmm1 +; SSE4-NEXT: movdqa %xmm10, %xmm0 +; SSE4-NEXT: blendvpd %xmm6, %xmm2 +; SSE4-NEXT: movdqa %xmm9, %xmm0 +; SSE4-NEXT: blendvpd %xmm7, %xmm3 +; SSE4-NEXT: movapd %xmm8, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test156: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm5, %xmm4 +; AVX1-NEXT: vpcmpeqd %xmm5, %xmm5, %xmm5 +; AVX1-NEXT: vpxor %xmm5, %xmm4, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm1, %xmm3, %xmm6 +; AVX1-NEXT: vpxor %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6 +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm7 +; AVX1-NEXT: vpcmpgtq %xmm6, %xmm7, %xmm6 +; AVX1-NEXT: vpxor %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm0, %xmm2, %xmm7 +; AVX1-NEXT: vpxor %xmm5, %xmm7, %xmm5 +; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5 +; AVX1-NEXT: vblendvpd %ymm5, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vblendvpd %ymm4, %ymm3, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test156: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %ymm1, %ymm3, %ymm4 +; AVX2-NEXT: vpcmpeqd %ymm5, %ymm5, %ymm5 +; AVX2-NEXT: vpxor %ymm5, %ymm4, %ymm4 +; AVX2-NEXT: vpcmpgtq %ymm0, %ymm2, %ymm6 +; AVX2-NEXT: vpxor %ymm5, %ymm6, %ymm5 +; AVX2-NEXT: vblendvpd %ymm5, %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vblendvpd %ymm4, %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test156: -; AVX512F: vpminsq -} - -define void @test157(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i64>* - %ptr.b = bitcast i32* %gep.b to <8 x i64>* - %load.a = load <8 x i64>, <8 x i64>* %ptr.a, align 2 - %load.b = load <8 x i64>, <8 x i64>* %ptr.b, align 2 - %cmp = icmp ult <8 x i64> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i64> %load.b, <8 x i64> %load.a - store <8 x i64> %sel, <8 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminsq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp sge <8 x i64> %a, %b + %sel = select <8 x i1> %cmp, <8 x i64> %b, <8 x i64> %a + ret <8 x i64> %sel +} + +define <8 x i64> @test157(<8 x i64> %a, <8 x i64> %b) { +; SSE2-LABEL: test157: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm11 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm8 +; SSE2-NEXT: pxor %xmm11, %xmm8 +; SSE2-NEXT: movdqa %xmm7, %xmm9 +; SSE2-NEXT: pxor %xmm11, %xmm9 +; SSE2-NEXT: movdqa %xmm9, %xmm10 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm10[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm8, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3] +; SSE2-NEXT: pand %xmm12, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm10[1,1,3,3] +; SSE2-NEXT: por %xmm9, %xmm8 +; SSE2-NEXT: movdqa %xmm2, %xmm9 +; SSE2-NEXT: pxor %xmm11, %xmm9 +; SSE2-NEXT: movdqa %xmm6, %xmm10 +; SSE2-NEXT: pxor %xmm11, %xmm10 +; SSE2-NEXT: movdqa %xmm10, %xmm12 +; SSE2-NEXT: pcmpgtd %xmm9, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm9, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3] +; SSE2-NEXT: pand %xmm13, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm12[1,1,3,3] +; SSE2-NEXT: por %xmm10, %xmm9 +; SSE2-NEXT: movdqa %xmm1, %xmm10 +; SSE2-NEXT: pxor %xmm11, %xmm10 +; SSE2-NEXT: movdqa %xmm5, %xmm12 +; SSE2-NEXT: pxor %xmm11, %xmm12 +; SSE2-NEXT: movdqa %xmm12, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm10, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm10, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3] +; SSE2-NEXT: pand %xmm14, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm13[1,1,3,3] +; SSE2-NEXT: por %xmm12, %xmm10 +; SSE2-NEXT: movdqa %xmm0, %xmm12 +; SSE2-NEXT: pxor %xmm11, %xmm12 +; SSE2-NEXT: pxor %xmm4, %xmm11 +; SSE2-NEXT: movdqa %xmm11, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm12, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm12, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[1,1,3,3] +; SSE2-NEXT: pand %xmm14, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[1,1,3,3] +; SSE2-NEXT: por %xmm12, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm11 +; SSE2-NEXT: por %xmm4, %xmm11 +; SSE2-NEXT: pand %xmm10, %xmm5 +; SSE2-NEXT: pandn %xmm1, %xmm10 +; SSE2-NEXT: por %xmm5, %xmm10 +; SSE2-NEXT: pand %xmm9, %xmm6 +; SSE2-NEXT: pandn %xmm2, %xmm9 +; SSE2-NEXT: por %xmm6, %xmm9 +; SSE2-NEXT: pand %xmm8, %xmm7 +; SSE2-NEXT: pandn %xmm3, %xmm8 +; SSE2-NEXT: por %xmm7, %xmm8 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test157: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm8 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm3, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: movdqa %xmm7, %xmm9 +; SSE4-NEXT: pxor %xmm0, %xmm9 +; SSE4-NEXT: pcmpgtq %xmm10, %xmm9 +; SSE4-NEXT: movdqa %xmm2, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: movdqa %xmm6, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: pcmpgtq %xmm11, %xmm10 +; SSE4-NEXT: movdqa %xmm1, %xmm12 +; SSE4-NEXT: pxor %xmm0, %xmm12 +; SSE4-NEXT: movdqa %xmm5, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm12, %xmm11 +; SSE4-NEXT: movdqa %xmm8, %xmm12 +; SSE4-NEXT: pxor %xmm0, %xmm12 +; SSE4-NEXT: pxor %xmm4, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm12, %xmm0 +; SSE4-NEXT: blendvpd %xmm4, %xmm8 +; SSE4-NEXT: movdqa %xmm11, %xmm0 +; SSE4-NEXT: blendvpd %xmm5, %xmm1 +; SSE4-NEXT: movdqa %xmm10, %xmm0 +; SSE4-NEXT: blendvpd %xmm6, %xmm2 +; SSE4-NEXT: movdqa %xmm9, %xmm0 +; SSE4-NEXT: blendvpd %xmm7, %xmm3 +; SSE4-NEXT: movapd %xmm8, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test157: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vmovaps {{.*#+}} xmm5 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm5, %xmm4, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm6, %xmm4 +; AVX1-NEXT: vxorps %xmm5, %xmm1, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm3, %xmm7 +; AVX1-NEXT: vpcmpgtq %xmm6, %xmm7, %xmm6 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm7, %xmm7 +; AVX1-NEXT: vpcmpgtq %xmm6, %xmm7, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm0, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm2, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm7, %xmm5, %xmm5 +; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5 +; AVX1-NEXT: vblendvpd %ymm5, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vblendvpd %ymm4, %ymm3, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test157: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm4 +; AVX2-NEXT: vpxor %ymm4, %ymm1, %ymm5 +; AVX2-NEXT: vpxor %ymm4, %ymm3, %ymm6 +; AVX2-NEXT: vpcmpgtq %ymm5, %ymm6, %ymm5 +; AVX2-NEXT: vpxor %ymm4, %ymm0, %ymm6 +; AVX2-NEXT: vpxor %ymm4, %ymm2, %ymm4 +; AVX2-NEXT: vpcmpgtq %ymm6, %ymm4, %ymm4 +; AVX2-NEXT: vblendvpd %ymm4, %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vblendvpd %ymm5, %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test157: -; AVX512F: vpmaxuq -} - -define void @test158(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i64>* - %ptr.b = bitcast i32* %gep.b to <8 x i64>* - %load.a = load <8 x i64>, <8 x i64>* %ptr.a, align 2 - %load.b = load <8 x i64>, <8 x i64>* %ptr.b, align 2 - %cmp = icmp ule <8 x i64> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i64> %load.b, <8 x i64> %load.a - store <8 x i64> %sel, <8 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxuq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ult <8 x i64> %a, %b + %sel = select <8 x i1> %cmp, <8 x i64> %b, <8 x i64> %a + ret <8 x i64> %sel +} + +define <8 x i64> @test158(<8 x i64> %a, <8 x i64> %b) { +; SSE2-LABEL: test158: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: movdqa %xmm8, -{{[0-9]+}}(%rsp) # 16-byte Spill +; SSE2-NEXT: movdqa %xmm3, %xmm7 +; SSE2-NEXT: movdqa %xmm2, %xmm3 +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm0, %xmm9 +; SSE2-NEXT: movdqa {{.*#+}} xmm10 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: pxor %xmm10, %xmm8 +; SSE2-NEXT: movdqa %xmm7, %xmm0 +; SSE2-NEXT: pxor %xmm10, %xmm0 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm8, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE2-NEXT: pand %xmm12, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: movdqa %xmm12, %xmm8 +; SSE2-NEXT: pxor %xmm1, %xmm8 +; SSE2-NEXT: movdqa %xmm6, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm3, %xmm13 +; SSE2-NEXT: pxor %xmm10, %xmm13 +; SSE2-NEXT: movdqa %xmm13, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm11, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[1,1,3,3] +; SSE2-NEXT: pand %xmm15, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm14[1,1,3,3] +; SSE2-NEXT: por %xmm11, %xmm13 +; SSE2-NEXT: movdqa %xmm5, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm2, %xmm14 +; SSE2-NEXT: pxor %xmm10, %xmm14 +; SSE2-NEXT: movdqa %xmm14, %xmm15 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm15 +; SSE2-NEXT: pcmpeqd %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm15[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm14[1,1,3,3] +; SSE2-NEXT: pand %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm15[1,1,3,3] +; SSE2-NEXT: por %xmm14, %xmm15 +; SSE2-NEXT: movdqa %xmm4, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: pxor %xmm9, %xmm10 +; SSE2-NEXT: movdqa %xmm10, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: pcmpeqd %xmm11, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm14[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm13, %xmm10 +; SSE2-NEXT: pxor %xmm1, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm14[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm14 +; SSE2-NEXT: movdqa %xmm15, %xmm11 +; SSE2-NEXT: pxor %xmm1, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm1 +; SSE2-NEXT: pandn %xmm4, %xmm14 +; SSE2-NEXT: pandn %xmm9, %xmm1 +; SSE2-NEXT: por %xmm14, %xmm1 +; SSE2-NEXT: pandn %xmm5, %xmm15 +; SSE2-NEXT: pandn %xmm2, %xmm11 +; SSE2-NEXT: por %xmm15, %xmm11 +; SSE2-NEXT: pandn %xmm6, %xmm13 +; SSE2-NEXT: pandn %xmm3, %xmm10 +; SSE2-NEXT: por %xmm13, %xmm10 +; SSE2-NEXT: pandn -{{[0-9]+}}(%rsp), %xmm12 # 16-byte Folded Reload +; SSE2-NEXT: pandn %xmm7, %xmm8 +; SSE2-NEXT: por %xmm12, %xmm8 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: movdqa %xmm11, %xmm1 +; SSE2-NEXT: movdqa %xmm10, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test158: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm8 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm7, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: movdqa %xmm3, %xmm9 +; SSE4-NEXT: pxor %xmm0, %xmm9 +; SSE4-NEXT: pcmpgtq %xmm10, %xmm9 +; SSE4-NEXT: pcmpeqd %xmm12, %xmm12 +; SSE4-NEXT: pxor %xmm12, %xmm9 +; SSE4-NEXT: movdqa %xmm6, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: movdqa %xmm2, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: pcmpgtq %xmm11, %xmm10 +; SSE4-NEXT: pxor %xmm12, %xmm10 +; SSE4-NEXT: movdqa %xmm5, %xmm13 +; SSE4-NEXT: pxor %xmm0, %xmm13 +; SSE4-NEXT: movdqa %xmm1, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm13, %xmm11 +; SSE4-NEXT: pxor %xmm12, %xmm11 +; SSE4-NEXT: movdqa %xmm4, %xmm13 +; SSE4-NEXT: pxor %xmm0, %xmm13 +; SSE4-NEXT: pxor %xmm8, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm13, %xmm0 +; SSE4-NEXT: pxor %xmm12, %xmm0 +; SSE4-NEXT: blendvpd %xmm4, %xmm8 +; SSE4-NEXT: movdqa %xmm11, %xmm0 +; SSE4-NEXT: blendvpd %xmm5, %xmm1 +; SSE4-NEXT: movdqa %xmm10, %xmm0 +; SSE4-NEXT: blendvpd %xmm6, %xmm2 +; SSE4-NEXT: movdqa %xmm9, %xmm0 +; SSE4-NEXT: blendvpd %xmm7, %xmm3 +; SSE4-NEXT: movapd %xmm8, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test158: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4 +; AVX1-NEXT: vmovaps {{.*#+}} xmm5 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm5, %xmm4, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm6, %xmm4 +; AVX1-NEXT: vpcmpeqd %xmm8, %xmm8, %xmm8 +; AVX1-NEXT: vpxor %xmm8, %xmm4, %xmm4 +; AVX1-NEXT: vxorps %xmm5, %xmm3, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm1, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm7, %xmm6, %xmm6 +; AVX1-NEXT: vpxor %xmm8, %xmm6, %xmm6 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4 +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm7, %xmm7 +; AVX1-NEXT: vpcmpgtq %xmm6, %xmm7, %xmm6 +; AVX1-NEXT: vpxor %xmm8, %xmm6, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm2, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm0, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm7, %xmm5, %xmm5 +; AVX1-NEXT: vpxor %xmm8, %xmm5, %xmm5 +; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5 +; AVX1-NEXT: vblendvpd %ymm5, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vblendvpd %ymm4, %ymm3, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test158: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm4 +; AVX2-NEXT: vpxor %ymm4, %ymm3, %ymm5 +; AVX2-NEXT: vpxor %ymm4, %ymm1, %ymm6 +; AVX2-NEXT: vpcmpgtq %ymm5, %ymm6, %ymm5 +; AVX2-NEXT: vpcmpeqd %ymm6, %ymm6, %ymm6 +; AVX2-NEXT: vpxor %ymm6, %ymm5, %ymm5 +; AVX2-NEXT: vpxor %ymm4, %ymm2, %ymm7 +; AVX2-NEXT: vpxor %ymm4, %ymm0, %ymm4 +; AVX2-NEXT: vpcmpgtq %ymm7, %ymm4, %ymm4 +; AVX2-NEXT: vpxor %ymm6, %ymm4, %ymm4 +; AVX2-NEXT: vblendvpd %ymm4, %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vblendvpd %ymm5, %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test158: -; AVX512F: vpmaxuq -} - -define void @test159(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i64>* - %ptr.b = bitcast i32* %gep.b to <8 x i64>* - %load.a = load <8 x i64>, <8 x i64>* %ptr.a, align 2 - %load.b = load <8 x i64>, <8 x i64>* %ptr.b, align 2 - %cmp = icmp ugt <8 x i64> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i64> %load.b, <8 x i64> %load.a - store <8 x i64> %sel, <8 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpmaxuq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ule <8 x i64> %a, %b + %sel = select <8 x i1> %cmp, <8 x i64> %b, <8 x i64> %a + ret <8 x i64> %sel +} + +define <8 x i64> @test159(<8 x i64> %a, <8 x i64> %b) { +; SSE2-LABEL: test159: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm11 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: pxor %xmm11, %xmm8 +; SSE2-NEXT: movdqa %xmm3, %xmm9 +; SSE2-NEXT: pxor %xmm11, %xmm9 +; SSE2-NEXT: movdqa %xmm9, %xmm10 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm10[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm8, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3] +; SSE2-NEXT: pand %xmm12, %xmm9 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm10[1,1,3,3] +; SSE2-NEXT: por %xmm9, %xmm8 +; SSE2-NEXT: movdqa %xmm6, %xmm9 +; SSE2-NEXT: pxor %xmm11, %xmm9 +; SSE2-NEXT: movdqa %xmm2, %xmm10 +; SSE2-NEXT: pxor %xmm11, %xmm10 +; SSE2-NEXT: movdqa %xmm10, %xmm12 +; SSE2-NEXT: pcmpgtd %xmm9, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm9, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3] +; SSE2-NEXT: pand %xmm13, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm12[1,1,3,3] +; SSE2-NEXT: por %xmm10, %xmm9 +; SSE2-NEXT: movdqa %xmm5, %xmm10 +; SSE2-NEXT: pxor %xmm11, %xmm10 +; SSE2-NEXT: movdqa %xmm1, %xmm12 +; SSE2-NEXT: pxor %xmm11, %xmm12 +; SSE2-NEXT: movdqa %xmm12, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm10, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm10, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3] +; SSE2-NEXT: pand %xmm14, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm13[1,1,3,3] +; SSE2-NEXT: por %xmm12, %xmm10 +; SSE2-NEXT: movdqa %xmm4, %xmm12 +; SSE2-NEXT: pxor %xmm11, %xmm12 +; SSE2-NEXT: pxor %xmm0, %xmm11 +; SSE2-NEXT: movdqa %xmm11, %xmm13 +; SSE2-NEXT: pcmpgtd %xmm12, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm12, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[1,1,3,3] +; SSE2-NEXT: pand %xmm14, %xmm12 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[1,1,3,3] +; SSE2-NEXT: por %xmm12, %xmm11 +; SSE2-NEXT: pand %xmm11, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm11 +; SSE2-NEXT: por %xmm4, %xmm11 +; SSE2-NEXT: pand %xmm10, %xmm5 +; SSE2-NEXT: pandn %xmm1, %xmm10 +; SSE2-NEXT: por %xmm5, %xmm10 +; SSE2-NEXT: pand %xmm9, %xmm6 +; SSE2-NEXT: pandn %xmm2, %xmm9 +; SSE2-NEXT: por %xmm6, %xmm9 +; SSE2-NEXT: pand %xmm8, %xmm7 +; SSE2-NEXT: pandn %xmm3, %xmm8 +; SSE2-NEXT: por %xmm7, %xmm8 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm10, %xmm1 +; SSE2-NEXT: movdqa %xmm9, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test159: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm8 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm7, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: movdqa %xmm3, %xmm9 +; SSE4-NEXT: pxor %xmm0, %xmm9 +; SSE4-NEXT: pcmpgtq %xmm10, %xmm9 +; SSE4-NEXT: movdqa %xmm6, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: movdqa %xmm2, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: pcmpgtq %xmm11, %xmm10 +; SSE4-NEXT: movdqa %xmm5, %xmm12 +; SSE4-NEXT: pxor %xmm0, %xmm12 +; SSE4-NEXT: movdqa %xmm1, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm12, %xmm11 +; SSE4-NEXT: movdqa %xmm4, %xmm12 +; SSE4-NEXT: pxor %xmm0, %xmm12 +; SSE4-NEXT: pxor %xmm8, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm12, %xmm0 +; SSE4-NEXT: blendvpd %xmm4, %xmm8 +; SSE4-NEXT: movdqa %xmm11, %xmm0 +; SSE4-NEXT: blendvpd %xmm5, %xmm1 +; SSE4-NEXT: movdqa %xmm10, %xmm0 +; SSE4-NEXT: blendvpd %xmm6, %xmm2 +; SSE4-NEXT: movdqa %xmm9, %xmm0 +; SSE4-NEXT: blendvpd %xmm7, %xmm3 +; SSE4-NEXT: movapd %xmm8, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test159: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4 +; AVX1-NEXT: vmovaps {{.*#+}} xmm5 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm5, %xmm4, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm6, %xmm4 +; AVX1-NEXT: vxorps %xmm5, %xmm3, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm1, %xmm7 +; AVX1-NEXT: vpcmpgtq %xmm6, %xmm7, %xmm6 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4 +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm7, %xmm7 +; AVX1-NEXT: vpcmpgtq %xmm6, %xmm7, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm2, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm0, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm7, %xmm5, %xmm5 +; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5 +; AVX1-NEXT: vblendvpd %ymm5, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vblendvpd %ymm4, %ymm3, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test159: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm4 +; AVX2-NEXT: vpxor %ymm4, %ymm3, %ymm5 +; AVX2-NEXT: vpxor %ymm4, %ymm1, %ymm6 +; AVX2-NEXT: vpcmpgtq %ymm5, %ymm6, %ymm5 +; AVX2-NEXT: vpxor %ymm4, %ymm2, %ymm6 +; AVX2-NEXT: vpxor %ymm4, %ymm0, %ymm4 +; AVX2-NEXT: vpcmpgtq %ymm6, %ymm4, %ymm4 +; AVX2-NEXT: vblendvpd %ymm4, %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vblendvpd %ymm5, %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test159: -; AVX512F: vpminuq -} - -define void @test160(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <8 x i64>* - %ptr.b = bitcast i32* %gep.b to <8 x i64>* - %load.a = load <8 x i64>, <8 x i64>* %ptr.a, align 2 - %load.b = load <8 x i64>, <8 x i64>* %ptr.b, align 2 - %cmp = icmp uge <8 x i64> %load.a, %load.b - %sel = select <8 x i1> %cmp, <8 x i64> %load.b, <8 x i64> %load.a - store <8 x i64> %sel, <8 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminuq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp ugt <8 x i64> %a, %b + %sel = select <8 x i1> %cmp, <8 x i64> %b, <8 x i64> %a + ret <8 x i64> %sel +} + +define <8 x i64> @test160(<8 x i64> %a, <8 x i64> %b) { +; SSE2-LABEL: test160: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa %xmm7, %xmm11 +; SSE2-NEXT: movdqa %xmm11, -{{[0-9]+}}(%rsp) # 16-byte Spill +; SSE2-NEXT: movdqa %xmm3, %xmm7 +; SSE2-NEXT: movdqa %xmm2, %xmm3 +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm0, %xmm9 +; SSE2-NEXT: movdqa {{.*#+}} xmm10 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm7, %xmm8 +; SSE2-NEXT: pxor %xmm10, %xmm8 +; SSE2-NEXT: movdqa %xmm11, %xmm0 +; SSE2-NEXT: pxor %xmm10, %xmm0 +; SSE2-NEXT: movdqa %xmm0, %xmm11 +; SSE2-NEXT: pcmpgtd %xmm8, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm8, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE2-NEXT: pand %xmm12, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm12 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: movdqa %xmm12, %xmm8 +; SSE2-NEXT: pxor %xmm1, %xmm8 +; SSE2-NEXT: movdqa %xmm3, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm6, %xmm13 +; SSE2-NEXT: pxor %xmm10, %xmm13 +; SSE2-NEXT: movdqa %xmm13, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm14[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm11, %xmm13 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[1,1,3,3] +; SSE2-NEXT: pand %xmm15, %xmm11 +; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm14[1,1,3,3] +; SSE2-NEXT: por %xmm11, %xmm13 +; SSE2-NEXT: movdqa %xmm2, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: movdqa %xmm5, %xmm14 +; SSE2-NEXT: pxor %xmm10, %xmm14 +; SSE2-NEXT: movdqa %xmm14, %xmm15 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm15 +; SSE2-NEXT: pcmpeqd %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm15[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm14[1,1,3,3] +; SSE2-NEXT: pand %xmm11, %xmm14 +; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm15[1,1,3,3] +; SSE2-NEXT: por %xmm14, %xmm15 +; SSE2-NEXT: movdqa %xmm9, %xmm11 +; SSE2-NEXT: pxor %xmm10, %xmm11 +; SSE2-NEXT: pxor %xmm4, %xmm10 +; SSE2-NEXT: movdqa %xmm10, %xmm14 +; SSE2-NEXT: pcmpgtd %xmm11, %xmm14 +; SSE2-NEXT: pcmpeqd %xmm11, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm14[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3] +; SSE2-NEXT: pand %xmm11, %xmm0 +; SSE2-NEXT: movdqa %xmm13, %xmm10 +; SSE2-NEXT: pxor %xmm1, %xmm10 +; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm14[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm14 +; SSE2-NEXT: movdqa %xmm15, %xmm11 +; SSE2-NEXT: pxor %xmm1, %xmm11 +; SSE2-NEXT: pxor %xmm14, %xmm1 +; SSE2-NEXT: pandn %xmm4, %xmm14 +; SSE2-NEXT: pandn %xmm9, %xmm1 +; SSE2-NEXT: por %xmm14, %xmm1 +; SSE2-NEXT: pandn %xmm5, %xmm15 +; SSE2-NEXT: pandn %xmm2, %xmm11 +; SSE2-NEXT: por %xmm15, %xmm11 +; SSE2-NEXT: pandn %xmm6, %xmm13 +; SSE2-NEXT: pandn %xmm3, %xmm10 +; SSE2-NEXT: por %xmm13, %xmm10 +; SSE2-NEXT: pandn -{{[0-9]+}}(%rsp), %xmm12 # 16-byte Folded Reload +; SSE2-NEXT: pandn %xmm7, %xmm8 +; SSE2-NEXT: por %xmm12, %xmm8 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: movdqa %xmm11, %xmm1 +; SSE2-NEXT: movdqa %xmm10, %xmm2 +; SSE2-NEXT: movdqa %xmm8, %xmm3 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test160: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm8 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm3, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: movdqa %xmm7, %xmm9 +; SSE4-NEXT: pxor %xmm0, %xmm9 +; SSE4-NEXT: pcmpgtq %xmm10, %xmm9 +; SSE4-NEXT: pcmpeqd %xmm12, %xmm12 +; SSE4-NEXT: pxor %xmm12, %xmm9 +; SSE4-NEXT: movdqa %xmm2, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: movdqa %xmm6, %xmm10 +; SSE4-NEXT: pxor %xmm0, %xmm10 +; SSE4-NEXT: pcmpgtq %xmm11, %xmm10 +; SSE4-NEXT: pxor %xmm12, %xmm10 +; SSE4-NEXT: movdqa %xmm1, %xmm13 +; SSE4-NEXT: pxor %xmm0, %xmm13 +; SSE4-NEXT: movdqa %xmm5, %xmm11 +; SSE4-NEXT: pxor %xmm0, %xmm11 +; SSE4-NEXT: pcmpgtq %xmm13, %xmm11 +; SSE4-NEXT: pxor %xmm12, %xmm11 +; SSE4-NEXT: movdqa %xmm8, %xmm13 +; SSE4-NEXT: pxor %xmm0, %xmm13 +; SSE4-NEXT: pxor %xmm4, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm13, %xmm0 +; SSE4-NEXT: pxor %xmm12, %xmm0 +; SSE4-NEXT: blendvpd %xmm4, %xmm8 +; SSE4-NEXT: movdqa %xmm11, %xmm0 +; SSE4-NEXT: blendvpd %xmm5, %xmm1 +; SSE4-NEXT: movdqa %xmm10, %xmm0 +; SSE4-NEXT: blendvpd %xmm6, %xmm2 +; SSE4-NEXT: movdqa %xmm9, %xmm0 +; SSE4-NEXT: blendvpd %xmm7, %xmm3 +; SSE4-NEXT: movapd %xmm8, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test160: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vmovaps {{.*#+}} xmm5 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm5, %xmm4, %xmm4 +; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm6, %xmm4 +; AVX1-NEXT: vpcmpeqd %xmm8, %xmm8, %xmm8 +; AVX1-NEXT: vpxor %xmm8, %xmm4, %xmm4 +; AVX1-NEXT: vxorps %xmm5, %xmm1, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm3, %xmm6 +; AVX1-NEXT: vpcmpgtq %xmm7, %xmm6, %xmm6 +; AVX1-NEXT: vpxor %xmm8, %xmm6, %xmm6 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm6, %xmm6 +; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm7, %xmm7 +; AVX1-NEXT: vpcmpgtq %xmm6, %xmm7, %xmm6 +; AVX1-NEXT: vpxor %xmm8, %xmm6, %xmm6 +; AVX1-NEXT: vxorps %xmm5, %xmm0, %xmm7 +; AVX1-NEXT: vxorps %xmm5, %xmm2, %xmm5 +; AVX1-NEXT: vpcmpgtq %xmm7, %xmm5, %xmm5 +; AVX1-NEXT: vpxor %xmm8, %xmm5, %xmm5 +; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5 +; AVX1-NEXT: vblendvpd %ymm5, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vblendvpd %ymm4, %ymm3, %ymm1, %ymm1 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test160: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm4 +; AVX2-NEXT: vpxor %ymm4, %ymm1, %ymm5 +; AVX2-NEXT: vpxor %ymm4, %ymm3, %ymm6 +; AVX2-NEXT: vpcmpgtq %ymm5, %ymm6, %ymm5 +; AVX2-NEXT: vpcmpeqd %ymm6, %ymm6, %ymm6 +; AVX2-NEXT: vpxor %ymm6, %ymm5, %ymm5 +; AVX2-NEXT: vpxor %ymm4, %ymm0, %ymm7 +; AVX2-NEXT: vpxor %ymm4, %ymm2, %ymm4 +; AVX2-NEXT: vpcmpgtq %ymm7, %ymm4, %ymm4 +; AVX2-NEXT: vpxor %ymm6, %ymm4, %ymm4 +; AVX2-NEXT: vblendvpd %ymm4, %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vblendvpd %ymm5, %ymm3, %ymm1, %ymm1 +; AVX2-NEXT: retq +; ; AVX512F-LABEL: test160: -; AVX512F: vpminuq -} - -define void @test161(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i64>* - %ptr.b = bitcast i32* %gep.b to <4 x i64>* - %load.a = load <4 x i64>, <4 x i64>* %ptr.a, align 2 - %load.b = load <4 x i64>, <4 x i64>* %ptr.b, align 2 - %cmp = icmp slt <4 x i64> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i64> %load.a, <4 x i64> %load.b - store <4 x i64> %sel, <4 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test161: -; AVX512VL: vpminsq -} - -define void @test162(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i64>* - %ptr.b = bitcast i32* %gep.b to <4 x i64>* - %load.a = load <4 x i64>, <4 x i64>* %ptr.a, align 2 - %load.b = load <4 x i64>, <4 x i64>* %ptr.b, align 2 - %cmp = icmp sle <4 x i64> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i64> %load.a, <4 x i64> %load.b - store <4 x i64> %sel, <4 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test162: -; AVX512VL: vpminsq -} - -define void @test163(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i64>* - %ptr.b = bitcast i32* %gep.b to <4 x i64>* - %load.a = load <4 x i64>, <4 x i64>* %ptr.a, align 2 - %load.b = load <4 x i64>, <4 x i64>* %ptr.b, align 2 - %cmp = icmp sgt <4 x i64> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i64> %load.a, <4 x i64> %load.b - store <4 x i64> %sel, <4 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test163: -; AVX512VL: vpmaxsq -} - -define void @test164(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i64>* - %ptr.b = bitcast i32* %gep.b to <4 x i64>* - %load.a = load <4 x i64>, <4 x i64>* %ptr.a, align 2 - %load.b = load <4 x i64>, <4 x i64>* %ptr.b, align 2 - %cmp = icmp sge <4 x i64> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i64> %load.a, <4 x i64> %load.b - store <4 x i64> %sel, <4 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test164: -; AVX512VL: vpmaxsq -} - -define void @test165(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i64>* - %ptr.b = bitcast i32* %gep.b to <4 x i64>* - %load.a = load <4 x i64>, <4 x i64>* %ptr.a, align 2 - %load.b = load <4 x i64>, <4 x i64>* %ptr.b, align 2 - %cmp = icmp ult <4 x i64> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i64> %load.a, <4 x i64> %load.b - store <4 x i64> %sel, <4 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test165: -; AVX512VL: vpminuq -} - -define void @test166(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i64>* - %ptr.b = bitcast i32* %gep.b to <4 x i64>* - %load.a = load <4 x i64>, <4 x i64>* %ptr.a, align 2 - %load.b = load <4 x i64>, <4 x i64>* %ptr.b, align 2 - %cmp = icmp ule <4 x i64> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i64> %load.a, <4 x i64> %load.b - store <4 x i64> %sel, <4 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test166: -; AVX512VL: vpminuq -} - -define void @test167(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i64>* - %ptr.b = bitcast i32* %gep.b to <4 x i64>* - %load.a = load <4 x i64>, <4 x i64>* %ptr.a, align 2 - %load.b = load <4 x i64>, <4 x i64>* %ptr.b, align 2 - %cmp = icmp ugt <4 x i64> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i64> %load.a, <4 x i64> %load.b - store <4 x i64> %sel, <4 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test167: -; AVX512VL: vpmaxuq -} - -define void @test168(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i64>* - %ptr.b = bitcast i32* %gep.b to <4 x i64>* - %load.a = load <4 x i64>, <4 x i64>* %ptr.a, align 2 - %load.b = load <4 x i64>, <4 x i64>* %ptr.b, align 2 - %cmp = icmp uge <4 x i64> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i64> %load.a, <4 x i64> %load.b - store <4 x i64> %sel, <4 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test168: -; AVX512VL: vpmaxuq -} - -define void @test169(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i64>* - %ptr.b = bitcast i32* %gep.b to <4 x i64>* - %load.a = load <4 x i64>, <4 x i64>* %ptr.a, align 2 - %load.b = load <4 x i64>, <4 x i64>* %ptr.b, align 2 - %cmp = icmp slt <4 x i64> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i64> %load.b, <4 x i64> %load.a - store <4 x i64> %sel, <4 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test169: -; AVX512VL: vpmaxsq -} - -define void @test170(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i64>* - %ptr.b = bitcast i32* %gep.b to <4 x i64>* - %load.a = load <4 x i64>, <4 x i64>* %ptr.a, align 2 - %load.b = load <4 x i64>, <4 x i64>* %ptr.b, align 2 - %cmp = icmp sle <4 x i64> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i64> %load.b, <4 x i64> %load.a - store <4 x i64> %sel, <4 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test170: -; AVX512VL: vpmaxsq -} - -define void @test171(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i64>* - %ptr.b = bitcast i32* %gep.b to <4 x i64>* - %load.a = load <4 x i64>, <4 x i64>* %ptr.a, align 2 - %load.b = load <4 x i64>, <4 x i64>* %ptr.b, align 2 - %cmp = icmp sgt <4 x i64> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i64> %load.b, <4 x i64> %load.a - store <4 x i64> %sel, <4 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test171: -; AVX512VL: vpminsq -} - -define void @test172(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i64>* - %ptr.b = bitcast i32* %gep.b to <4 x i64>* - %load.a = load <4 x i64>, <4 x i64>* %ptr.a, align 2 - %load.b = load <4 x i64>, <4 x i64>* %ptr.b, align 2 - %cmp = icmp sge <4 x i64> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i64> %load.b, <4 x i64> %load.a - store <4 x i64> %sel, <4 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test172: -; AVX512VL: vpminsq -} - -define void @test173(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i64>* - %ptr.b = bitcast i32* %gep.b to <4 x i64>* - %load.a = load <4 x i64>, <4 x i64>* %ptr.a, align 2 - %load.b = load <4 x i64>, <4 x i64>* %ptr.b, align 2 - %cmp = icmp ult <4 x i64> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i64> %load.b, <4 x i64> %load.a - store <4 x i64> %sel, <4 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test173: -; AVX512VL: vpmaxuq -} - -define void @test174(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i64>* - %ptr.b = bitcast i32* %gep.b to <4 x i64>* - %load.a = load <4 x i64>, <4 x i64>* %ptr.a, align 2 - %load.b = load <4 x i64>, <4 x i64>* %ptr.b, align 2 - %cmp = icmp ule <4 x i64> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i64> %load.b, <4 x i64> %load.a - store <4 x i64> %sel, <4 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test174: -; AVX512VL: vpmaxuq -} - -define void @test175(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i64>* - %ptr.b = bitcast i32* %gep.b to <4 x i64>* - %load.a = load <4 x i64>, <4 x i64>* %ptr.a, align 2 - %load.b = load <4 x i64>, <4 x i64>* %ptr.b, align 2 - %cmp = icmp ugt <4 x i64> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i64> %load.b, <4 x i64> %load.a - store <4 x i64> %sel, <4 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test175: -; AVX512VL: vpminuq -} - -define void @test176(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <4 x i64>* - %ptr.b = bitcast i32* %gep.b to <4 x i64>* - %load.a = load <4 x i64>, <4 x i64>* %ptr.a, align 2 - %load.b = load <4 x i64>, <4 x i64>* %ptr.b, align 2 - %cmp = icmp uge <4 x i64> %load.a, %load.b - %sel = select <4 x i1> %cmp, <4 x i64> %load.b, <4 x i64> %load.a - store <4 x i64> %sel, <4 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test176: -; AVX512VL: vpminuq -} - -define void @test177(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <2 x i64>* - %ptr.b = bitcast i32* %gep.b to <2 x i64>* - %load.a = load <2 x i64>, <2 x i64>* %ptr.a, align 2 - %load.b = load <2 x i64>, <2 x i64>* %ptr.b, align 2 - %cmp = icmp slt <2 x i64> %load.a, %load.b - %sel = select <2 x i1> %cmp, <2 x i64> %load.a, <2 x i64> %load.b - store <2 x i64> %sel, <2 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test177: -; AVX512VL: vpminsq -} - -define void @test178(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <2 x i64>* - %ptr.b = bitcast i32* %gep.b to <2 x i64>* - %load.a = load <2 x i64>, <2 x i64>* %ptr.a, align 2 - %load.b = load <2 x i64>, <2 x i64>* %ptr.b, align 2 - %cmp = icmp sle <2 x i64> %load.a, %load.b - %sel = select <2 x i1> %cmp, <2 x i64> %load.a, <2 x i64> %load.b - store <2 x i64> %sel, <2 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test178: -; AVX512VL: vpminsq -} - -define void @test179(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <2 x i64>* - %ptr.b = bitcast i32* %gep.b to <2 x i64>* - %load.a = load <2 x i64>, <2 x i64>* %ptr.a, align 2 - %load.b = load <2 x i64>, <2 x i64>* %ptr.b, align 2 - %cmp = icmp sgt <2 x i64> %load.a, %load.b - %sel = select <2 x i1> %cmp, <2 x i64> %load.a, <2 x i64> %load.b - store <2 x i64> %sel, <2 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test179: -; AVX512VL: vpmaxsq -} - -define void @test180(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <2 x i64>* - %ptr.b = bitcast i32* %gep.b to <2 x i64>* - %load.a = load <2 x i64>, <2 x i64>* %ptr.a, align 2 - %load.b = load <2 x i64>, <2 x i64>* %ptr.b, align 2 - %cmp = icmp sge <2 x i64> %load.a, %load.b - %sel = select <2 x i1> %cmp, <2 x i64> %load.a, <2 x i64> %load.b - store <2 x i64> %sel, <2 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test180: -; AVX512VL: vpmaxsq -} - -define void @test181(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <2 x i64>* - %ptr.b = bitcast i32* %gep.b to <2 x i64>* - %load.a = load <2 x i64>, <2 x i64>* %ptr.a, align 2 - %load.b = load <2 x i64>, <2 x i64>* %ptr.b, align 2 - %cmp = icmp ult <2 x i64> %load.a, %load.b - %sel = select <2 x i1> %cmp, <2 x i64> %load.a, <2 x i64> %load.b - store <2 x i64> %sel, <2 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test181: -; AVX512VL: vpminuq -} - -define void @test182(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <2 x i64>* - %ptr.b = bitcast i32* %gep.b to <2 x i64>* - %load.a = load <2 x i64>, <2 x i64>* %ptr.a, align 2 - %load.b = load <2 x i64>, <2 x i64>* %ptr.b, align 2 - %cmp = icmp ule <2 x i64> %load.a, %load.b - %sel = select <2 x i1> %cmp, <2 x i64> %load.a, <2 x i64> %load.b - store <2 x i64> %sel, <2 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test182: -; AVX512VL: vpminuq -} - -define void @test183(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <2 x i64>* - %ptr.b = bitcast i32* %gep.b to <2 x i64>* - %load.a = load <2 x i64>, <2 x i64>* %ptr.a, align 2 - %load.b = load <2 x i64>, <2 x i64>* %ptr.b, align 2 - %cmp = icmp ugt <2 x i64> %load.a, %load.b - %sel = select <2 x i1> %cmp, <2 x i64> %load.a, <2 x i64> %load.b - store <2 x i64> %sel, <2 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test183: -; AVX512VL: vpmaxuq -} - -define void @test184(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <2 x i64>* - %ptr.b = bitcast i32* %gep.b to <2 x i64>* - %load.a = load <2 x i64>, <2 x i64>* %ptr.a, align 2 - %load.b = load <2 x i64>, <2 x i64>* %ptr.b, align 2 - %cmp = icmp uge <2 x i64> %load.a, %load.b - %sel = select <2 x i1> %cmp, <2 x i64> %load.a, <2 x i64> %load.b - store <2 x i64> %sel, <2 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test184: -; AVX512VL: vpmaxuq -} - -define void @test185(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <2 x i64>* - %ptr.b = bitcast i32* %gep.b to <2 x i64>* - %load.a = load <2 x i64>, <2 x i64>* %ptr.a, align 2 - %load.b = load <2 x i64>, <2 x i64>* %ptr.b, align 2 - %cmp = icmp slt <2 x i64> %load.a, %load.b - %sel = select <2 x i1> %cmp, <2 x i64> %load.b, <2 x i64> %load.a - store <2 x i64> %sel, <2 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test185: -; AVX512VL: vpmaxsq -} - -define void @test186(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <2 x i64>* - %ptr.b = bitcast i32* %gep.b to <2 x i64>* - %load.a = load <2 x i64>, <2 x i64>* %ptr.a, align 2 - %load.b = load <2 x i64>, <2 x i64>* %ptr.b, align 2 - %cmp = icmp sle <2 x i64> %load.a, %load.b - %sel = select <2 x i1> %cmp, <2 x i64> %load.b, <2 x i64> %load.a - store <2 x i64> %sel, <2 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test186: -; AVX512VL: vpmaxsq -} - -define void @test187(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <2 x i64>* - %ptr.b = bitcast i32* %gep.b to <2 x i64>* - %load.a = load <2 x i64>, <2 x i64>* %ptr.a, align 2 - %load.b = load <2 x i64>, <2 x i64>* %ptr.b, align 2 - %cmp = icmp sgt <2 x i64> %load.a, %load.b - %sel = select <2 x i1> %cmp, <2 x i64> %load.b, <2 x i64> %load.a - store <2 x i64> %sel, <2 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test187: -; AVX512VL: vpminsq -} - -define void @test188(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <2 x i64>* - %ptr.b = bitcast i32* %gep.b to <2 x i64>* - %load.a = load <2 x i64>, <2 x i64>* %ptr.a, align 2 - %load.b = load <2 x i64>, <2 x i64>* %ptr.b, align 2 - %cmp = icmp sge <2 x i64> %load.a, %load.b - %sel = select <2 x i1> %cmp, <2 x i64> %load.b, <2 x i64> %load.a - store <2 x i64> %sel, <2 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test188: -; AVX512VL: vpminsq -} - -define void @test189(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <2 x i64>* - %ptr.b = bitcast i32* %gep.b to <2 x i64>* - %load.a = load <2 x i64>, <2 x i64>* %ptr.a, align 2 - %load.b = load <2 x i64>, <2 x i64>* %ptr.b, align 2 - %cmp = icmp ult <2 x i64> %load.a, %load.b - %sel = select <2 x i1> %cmp, <2 x i64> %load.b, <2 x i64> %load.a - store <2 x i64> %sel, <2 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test189: -; AVX512VL: vpmaxuq -} - -define void @test190(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <2 x i64>* - %ptr.b = bitcast i32* %gep.b to <2 x i64>* - %load.a = load <2 x i64>, <2 x i64>* %ptr.a, align 2 - %load.b = load <2 x i64>, <2 x i64>* %ptr.b, align 2 - %cmp = icmp ule <2 x i64> %load.a, %load.b - %sel = select <2 x i1> %cmp, <2 x i64> %load.b, <2 x i64> %load.a - store <2 x i64> %sel, <2 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test190: -; AVX512VL: vpmaxuq -} - -define void @test191(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <2 x i64>* - %ptr.b = bitcast i32* %gep.b to <2 x i64>* - %load.a = load <2 x i64>, <2 x i64>* %ptr.a, align 2 - %load.b = load <2 x i64>, <2 x i64>* %ptr.b, align 2 - %cmp = icmp ugt <2 x i64> %load.a, %load.b - %sel = select <2 x i1> %cmp, <2 x i64> %load.b, <2 x i64> %load.a - store <2 x i64> %sel, <2 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test191: -; AVX512VL: vpminuq -} - -define void @test192(i32* nocapture %a, i32* nocapture %b) nounwind { -vector.ph: - br label %vector.body - -vector.body: ; preds = %vector.body, %vector.ph - %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] - %gep.a = getelementptr inbounds i32, i32* %a, i64 %index - %gep.b = getelementptr inbounds i32, i32* %b, i64 %index - %ptr.a = bitcast i32* %gep.a to <2 x i64>* - %ptr.b = bitcast i32* %gep.b to <2 x i64>* - %load.a = load <2 x i64>, <2 x i64>* %ptr.a, align 2 - %load.b = load <2 x i64>, <2 x i64>* %ptr.b, align 2 - %cmp = icmp uge <2 x i64> %load.a, %load.b - %sel = select <2 x i1> %cmp, <2 x i64> %load.b, <2 x i64> %load.a - store <2 x i64> %sel, <2 x i64>* %ptr.a, align 2 - %index.next = add i64 %index, 8 - %loop = icmp eq i64 %index.next, 16384 - br i1 %loop, label %for.end, label %vector.body - -for.end: ; preds = %vector.body - ret void - -; AVX512VL-LABEL: test192: -; AVX512VL: vpminuq +; AVX512F: # BB#0: # %entry +; AVX512F-NEXT: vpminuq %zmm1, %zmm0, %zmm0 +; AVX512F-NEXT: retq +entry: + %cmp = icmp uge <8 x i64> %a, %b + %sel = select <8 x i1> %cmp, <8 x i64> %b, <8 x i64> %a + ret <8 x i64> %sel +} + +define <4 x i64> @test161(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: test161: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: movdqa %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm5, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: pxor %xmm2, %xmm4 +; SSE2-NEXT: movdqa %xmm4, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm5, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: pand %xmm6, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm6 +; SSE2-NEXT: por %xmm6, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test161: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm4 +; SSE4-NEXT: movdqa %xmm3, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm1, %xmm5 +; SSE4-NEXT: movdqa %xmm2, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm4, %xmm0 +; SSE4-NEXT: blendvpd %xmm4, %xmm2 +; SSE4-NEXT: movdqa %xmm5, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm3 +; SSE4-NEXT: movapd %xmm2, %xmm0 +; SSE4-NEXT: movapd %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test161: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test161: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test161: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminsq %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp slt <4 x i64> %a, %b + %sel = select <4 x i1> %cmp, <4 x i64> %a, <4 x i64> %b + ret <4 x i64> %sel +} + +define <4 x i64> @test162(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: test162: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pxor %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm8 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm8, %xmm9 +; SSE2-NEXT: pxor %xmm4, %xmm9 +; SSE2-NEXT: movdqa %xmm2, %xmm6 +; SSE2-NEXT: pxor %xmm7, %xmm6 +; SSE2-NEXT: pxor %xmm0, %xmm7 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm5[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: pand %xmm10, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm1, %xmm8 +; SSE2-NEXT: pandn %xmm3, %xmm9 +; SSE2-NEXT: por %xmm8, %xmm9 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm9, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test162: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm4 +; SSE4-NEXT: movdqa %xmm1, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm3, %xmm5 +; SSE4-NEXT: pcmpeqd %xmm6, %xmm6 +; SSE4-NEXT: pxor %xmm6, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm2, %xmm0 +; SSE4-NEXT: pxor %xmm6, %xmm0 +; SSE4-NEXT: blendvpd %xmm4, %xmm2 +; SSE4-NEXT: movdqa %xmm5, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm3 +; SSE4-NEXT: movapd %xmm2, %xmm0 +; SSE4-NEXT: movapd %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test162: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm4 +; AVX1-NEXT: vpxor %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test162: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm2 +; AVX2-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3 +; AVX2-NEXT: vpxor %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test162: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminsq %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sle <4 x i64> %a, %b + %sel = select <4 x i1> %cmp, <4 x i64> %a, <4 x i64> %b + ret <4 x i64> %sel +} + +define <4 x i64> @test163(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: test163: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm3, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm1, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: movdqa %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm5, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm2, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: pxor %xmm0, %xmm4 +; SSE2-NEXT: movdqa %xmm4, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm5, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: pand %xmm6, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm6 +; SSE2-NEXT: por %xmm6, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test163: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm4 +; SSE4-NEXT: movdqa %xmm1, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm3, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm2, %xmm0 +; SSE4-NEXT: blendvpd %xmm4, %xmm2 +; SSE4-NEXT: movdqa %xmm5, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm3 +; SSE4-NEXT: movapd %xmm2, %xmm0 +; SSE4-NEXT: movapd %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test163: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test163: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test163: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxsq %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sgt <4 x i64> %a, %b + %sel = select <4 x i1> %cmp, <4 x i64> %a, <4 x i64> %b + ret <4 x i64> %sel +} + +define <4 x i64> @test164(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: test164: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm3, %xmm5 +; SSE2-NEXT: pxor %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm8 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm8, %xmm9 +; SSE2-NEXT: pxor %xmm4, %xmm9 +; SSE2-NEXT: movdqa %xmm0, %xmm6 +; SSE2-NEXT: pxor %xmm7, %xmm6 +; SSE2-NEXT: pxor %xmm2, %xmm7 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm5[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: pand %xmm10, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm1, %xmm8 +; SSE2-NEXT: pandn %xmm3, %xmm9 +; SSE2-NEXT: por %xmm8, %xmm9 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm9, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test164: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm4 +; SSE4-NEXT: movdqa %xmm3, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm1, %xmm5 +; SSE4-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE4-NEXT: pxor %xmm0, %xmm5 +; SSE4-NEXT: movdqa %xmm2, %xmm6 +; SSE4-NEXT: pcmpgtq %xmm4, %xmm6 +; SSE4-NEXT: pxor %xmm6, %xmm0 +; SSE4-NEXT: blendvpd %xmm4, %xmm2 +; SSE4-NEXT: movdqa %xmm5, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm3 +; SSE4-NEXT: movapd %xmm2, %xmm0 +; SSE4-NEXT: movapd %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test164: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm4 +; AVX1-NEXT: vpxor %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test164: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm2 +; AVX2-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3 +; AVX2-NEXT: vpxor %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test164: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxsq %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sge <4 x i64> %a, %b + %sel = select <4 x i1> %cmp, <4 x i64> %a, <4 x i64> %b + ret <4 x i64> %sel +} + +define <4 x i64> @test165(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: test165: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: movdqa %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm5, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm0, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: pxor %xmm2, %xmm4 +; SSE2-NEXT: movdqa %xmm4, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm5, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: pand %xmm6, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm6 +; SSE2-NEXT: por %xmm6, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test165: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm4 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm1, %xmm6 +; SSE4-NEXT: pxor %xmm0, %xmm6 +; SSE4-NEXT: movdqa %xmm3, %xmm5 +; SSE4-NEXT: pxor %xmm0, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm6, %xmm5 +; SSE4-NEXT: movdqa %xmm4, %xmm6 +; SSE4-NEXT: pxor %xmm0, %xmm6 +; SSE4-NEXT: pxor %xmm2, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm6, %xmm0 +; SSE4-NEXT: blendvpd %xmm4, %xmm2 +; SSE4-NEXT: movdqa %xmm5, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm3 +; SSE4-NEXT: movapd %xmm2, %xmm0 +; SSE4-NEXT: movapd %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test165: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm4, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vxorps %xmm3, %xmm0, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test165: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm3 +; AVX2-NEXT: vpxor %ymm2, %ymm1, %ymm2 +; AVX2-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test165: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminuq %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ult <4 x i64> %a, %b + %sel = select <4 x i1> %cmp, <4 x i64> %a, <4 x i64> %b + ret <4 x i64> %sel +} + +define <4 x i64> @test166(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: test166: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pxor %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm8 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm8, %xmm9 +; SSE2-NEXT: pxor %xmm4, %xmm9 +; SSE2-NEXT: movdqa %xmm2, %xmm6 +; SSE2-NEXT: pxor %xmm7, %xmm6 +; SSE2-NEXT: pxor %xmm0, %xmm7 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm5[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: pand %xmm10, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm1, %xmm8 +; SSE2-NEXT: pandn %xmm3, %xmm9 +; SSE2-NEXT: por %xmm8, %xmm9 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm9, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test166: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm4 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm3, %xmm6 +; SSE4-NEXT: pxor %xmm0, %xmm6 +; SSE4-NEXT: movdqa %xmm1, %xmm5 +; SSE4-NEXT: pxor %xmm0, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm6, %xmm5 +; SSE4-NEXT: pcmpeqd %xmm6, %xmm6 +; SSE4-NEXT: pxor %xmm6, %xmm5 +; SSE4-NEXT: movdqa %xmm2, %xmm7 +; SSE4-NEXT: pxor %xmm0, %xmm7 +; SSE4-NEXT: pxor %xmm4, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm7, %xmm0 +; SSE4-NEXT: pxor %xmm6, %xmm0 +; SSE4-NEXT: blendvpd %xmm4, %xmm2 +; SSE4-NEXT: movdqa %xmm5, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm3 +; SSE4-NEXT: movapd %xmm2, %xmm0 +; SSE4-NEXT: movapd %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test166: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm4, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm4, %xmm4, %xmm4 +; AVX1-NEXT: vpxor %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vxorps %xmm3, %xmm1, %xmm5 +; AVX1-NEXT: vxorps %xmm3, %xmm0, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm5, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test166: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpxor %ymm2, %ymm1, %ymm3 +; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm2 +; AVX2-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3 +; AVX2-NEXT: vpxor %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test166: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminuq %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ule <4 x i64> %a, %b + %sel = select <4 x i1> %cmp, <4 x i64> %a, <4 x i64> %b + ret <4 x i64> %sel +} + +define <4 x i64> @test167(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: test167: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: movdqa %xmm1, %xmm6 +; SSE2-NEXT: pxor %xmm4, %xmm6 +; SSE2-NEXT: movdqa %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm5, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm2, %xmm5 +; SSE2-NEXT: pxor %xmm4, %xmm5 +; SSE2-NEXT: pxor %xmm0, %xmm4 +; SSE2-NEXT: movdqa %xmm4, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm5, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm5, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm0 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: por %xmm5, %xmm0 +; SSE2-NEXT: pand %xmm6, %xmm1 +; SSE2-NEXT: pandn %xmm3, %xmm6 +; SSE2-NEXT: por %xmm6, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test167: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm4 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm3, %xmm6 +; SSE4-NEXT: pxor %xmm0, %xmm6 +; SSE4-NEXT: movdqa %xmm1, %xmm5 +; SSE4-NEXT: pxor %xmm0, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm6, %xmm5 +; SSE4-NEXT: movdqa %xmm2, %xmm6 +; SSE4-NEXT: pxor %xmm0, %xmm6 +; SSE4-NEXT: pxor %xmm4, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm6, %xmm0 +; SSE4-NEXT: blendvpd %xmm4, %xmm2 +; SSE4-NEXT: movdqa %xmm5, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm3 +; SSE4-NEXT: movapd %xmm2, %xmm0 +; SSE4-NEXT: movapd %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test167: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm4, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vxorps %xmm3, %xmm1, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm0, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test167: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpxor %ymm2, %ymm1, %ymm3 +; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm2 +; AVX2-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test167: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxuq %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ugt <4 x i64> %a, %b + %sel = select <4 x i1> %cmp, <4 x i64> %a, <4 x i64> %b + ret <4 x i64> %sel +} + +define <4 x i64> @test168(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: test168: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm3, %xmm5 +; SSE2-NEXT: pxor %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm8 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm8, %xmm9 +; SSE2-NEXT: pxor %xmm4, %xmm9 +; SSE2-NEXT: movdqa %xmm0, %xmm6 +; SSE2-NEXT: pxor %xmm7, %xmm6 +; SSE2-NEXT: pxor %xmm2, %xmm7 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm5[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: pand %xmm10, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: pandn %xmm2, %xmm4 +; SSE2-NEXT: por %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm1, %xmm8 +; SSE2-NEXT: pandn %xmm3, %xmm9 +; SSE2-NEXT: por %xmm8, %xmm9 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm9, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test168: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm4 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm1, %xmm6 +; SSE4-NEXT: pxor %xmm0, %xmm6 +; SSE4-NEXT: movdqa %xmm3, %xmm5 +; SSE4-NEXT: pxor %xmm0, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm6, %xmm5 +; SSE4-NEXT: pcmpeqd %xmm6, %xmm6 +; SSE4-NEXT: pxor %xmm6, %xmm5 +; SSE4-NEXT: movdqa %xmm4, %xmm7 +; SSE4-NEXT: pxor %xmm0, %xmm7 +; SSE4-NEXT: pxor %xmm2, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm7, %xmm0 +; SSE4-NEXT: pxor %xmm6, %xmm0 +; SSE4-NEXT: blendvpd %xmm4, %xmm2 +; SSE4-NEXT: movdqa %xmm5, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm3 +; SSE4-NEXT: movapd %xmm2, %xmm0 +; SSE4-NEXT: movapd %xmm3, %xmm1 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test168: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm4, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm4, %xmm4, %xmm4 +; AVX1-NEXT: vpxor %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vxorps %xmm3, %xmm0, %xmm5 +; AVX1-NEXT: vxorps %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm5, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test168: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm3 +; AVX2-NEXT: vpxor %ymm2, %ymm1, %ymm2 +; AVX2-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3 +; AVX2-NEXT: vpxor %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm0, %ymm1, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test168: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxuq %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp uge <4 x i64> %a, %b + %sel = select <4 x i1> %cmp, <4 x i64> %a, <4 x i64> %b + ret <4 x i64> %sel +} + +define <4 x i64> @test169(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: test169: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pxor %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm6 +; SSE2-NEXT: pxor %xmm5, %xmm6 +; SSE2-NEXT: pxor %xmm2, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm6, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: por %xmm2, %xmm5 +; SSE2-NEXT: pand %xmm4, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm4 +; SSE2-NEXT: por %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test169: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm4 +; SSE4-NEXT: movdqa %xmm3, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm1, %xmm5 +; SSE4-NEXT: movdqa %xmm2, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm4, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm4 +; SSE4-NEXT: movdqa %xmm5, %xmm0 +; SSE4-NEXT: blendvpd %xmm3, %xmm1 +; SSE4-NEXT: movapd %xmm4, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test169: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test169: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test169: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxsq %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp slt <4 x i64> %a, %b + %sel = select <4 x i1> %cmp, <4 x i64> %b, <4 x i64> %a + ret <4 x i64> %sel +} + +define <4 x i64> @test170(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: test170: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pxor %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm8 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm8, %xmm9 +; SSE2-NEXT: pxor %xmm4, %xmm9 +; SSE2-NEXT: movdqa %xmm2, %xmm6 +; SSE2-NEXT: pxor %xmm7, %xmm6 +; SSE2-NEXT: pxor %xmm0, %xmm7 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm5[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: pand %xmm10, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: pandn %xmm0, %xmm4 +; SSE2-NEXT: por %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm3, %xmm8 +; SSE2-NEXT: pandn %xmm1, %xmm9 +; SSE2-NEXT: por %xmm8, %xmm9 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm9, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test170: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm4 +; SSE4-NEXT: movdqa %xmm1, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm3, %xmm5 +; SSE4-NEXT: pcmpeqd %xmm6, %xmm6 +; SSE4-NEXT: pxor %xmm6, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm2, %xmm0 +; SSE4-NEXT: pxor %xmm6, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm4 +; SSE4-NEXT: movdqa %xmm5, %xmm0 +; SSE4-NEXT: blendvpd %xmm3, %xmm1 +; SSE4-NEXT: movapd %xmm4, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test170: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm4 +; AVX1-NEXT: vpxor %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test170: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm2 +; AVX2-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3 +; AVX2-NEXT: vpxor %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test170: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxsq %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sle <4 x i64> %a, %b + %sel = select <4 x i1> %cmp, <4 x i64> %b, <4 x i64> %a + ret <4 x i64> %sel +} + +define <4 x i64> @test171(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: test171: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: movdqa %xmm1, %xmm6 +; SSE2-NEXT: pxor %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm2, %xmm6 +; SSE2-NEXT: pxor %xmm5, %xmm6 +; SSE2-NEXT: pxor %xmm0, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm6, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: por %xmm2, %xmm5 +; SSE2-NEXT: pand %xmm4, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm4 +; SSE2-NEXT: por %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test171: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm4 +; SSE4-NEXT: movdqa %xmm1, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm3, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm2, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm4 +; SSE4-NEXT: movdqa %xmm5, %xmm0 +; SSE4-NEXT: blendvpd %xmm3, %xmm1 +; SSE4-NEXT: movapd %xmm4, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test171: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test171: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test171: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminsq %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sgt <4 x i64> %a, %b + %sel = select <4 x i1> %cmp, <4 x i64> %b, <4 x i64> %a + ret <4 x i64> %sel +} + +define <4 x i64> @test172(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: test172: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm3, %xmm5 +; SSE2-NEXT: pxor %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm8 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm8, %xmm9 +; SSE2-NEXT: pxor %xmm4, %xmm9 +; SSE2-NEXT: movdqa %xmm0, %xmm6 +; SSE2-NEXT: pxor %xmm7, %xmm6 +; SSE2-NEXT: pxor %xmm2, %xmm7 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm5[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: pand %xmm10, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: pandn %xmm0, %xmm4 +; SSE2-NEXT: por %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm3, %xmm8 +; SSE2-NEXT: pandn %xmm1, %xmm9 +; SSE2-NEXT: por %xmm8, %xmm9 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm9, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test172: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm4 +; SSE4-NEXT: movdqa %xmm3, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm1, %xmm5 +; SSE4-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE4-NEXT: pxor %xmm0, %xmm5 +; SSE4-NEXT: movdqa %xmm2, %xmm6 +; SSE4-NEXT: pcmpgtq %xmm4, %xmm6 +; SSE4-NEXT: pxor %xmm6, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm4 +; SSE4-NEXT: movdqa %xmm5, %xmm0 +; SSE4-NEXT: blendvpd %xmm3, %xmm1 +; SSE4-NEXT: movapd %xmm4, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test172: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm4 +; AVX1-NEXT: vpxor %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test172: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm2 +; AVX2-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3 +; AVX2-NEXT: vpxor %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test172: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminsq %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sge <4 x i64> %a, %b + %sel = select <4 x i1> %cmp, <4 x i64> %b, <4 x i64> %a + ret <4 x i64> %sel +} + +define <4 x i64> @test173(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: test173: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: movdqa %xmm3, %xmm6 +; SSE2-NEXT: pxor %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm0, %xmm6 +; SSE2-NEXT: pxor %xmm5, %xmm6 +; SSE2-NEXT: pxor %xmm2, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm6, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: por %xmm2, %xmm5 +; SSE2-NEXT: pand %xmm4, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm4 +; SSE2-NEXT: por %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test173: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm4 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm1, %xmm6 +; SSE4-NEXT: pxor %xmm0, %xmm6 +; SSE4-NEXT: movdqa %xmm3, %xmm5 +; SSE4-NEXT: pxor %xmm0, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm6, %xmm5 +; SSE4-NEXT: movdqa %xmm4, %xmm6 +; SSE4-NEXT: pxor %xmm0, %xmm6 +; SSE4-NEXT: pxor %xmm2, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm6, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm4 +; SSE4-NEXT: movdqa %xmm5, %xmm0 +; SSE4-NEXT: blendvpd %xmm3, %xmm1 +; SSE4-NEXT: movapd %xmm4, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test173: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm4, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vxorps %xmm3, %xmm0, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test173: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm3 +; AVX2-NEXT: vpxor %ymm2, %ymm1, %ymm2 +; AVX2-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test173: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxuq %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ult <4 x i64> %a, %b + %sel = select <4 x i1> %cmp, <4 x i64> %b, <4 x i64> %a + ret <4 x i64> %sel +} + +define <4 x i64> @test174(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: test174: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm1, %xmm5 +; SSE2-NEXT: pxor %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm8 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm8, %xmm9 +; SSE2-NEXT: pxor %xmm4, %xmm9 +; SSE2-NEXT: movdqa %xmm2, %xmm6 +; SSE2-NEXT: pxor %xmm7, %xmm6 +; SSE2-NEXT: pxor %xmm0, %xmm7 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm5[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: pand %xmm10, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: pandn %xmm0, %xmm4 +; SSE2-NEXT: por %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm3, %xmm8 +; SSE2-NEXT: pandn %xmm1, %xmm9 +; SSE2-NEXT: por %xmm8, %xmm9 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm9, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test174: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm4 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm3, %xmm6 +; SSE4-NEXT: pxor %xmm0, %xmm6 +; SSE4-NEXT: movdqa %xmm1, %xmm5 +; SSE4-NEXT: pxor %xmm0, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm6, %xmm5 +; SSE4-NEXT: pcmpeqd %xmm6, %xmm6 +; SSE4-NEXT: pxor %xmm6, %xmm5 +; SSE4-NEXT: movdqa %xmm2, %xmm7 +; SSE4-NEXT: pxor %xmm0, %xmm7 +; SSE4-NEXT: pxor %xmm4, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm7, %xmm0 +; SSE4-NEXT: pxor %xmm6, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm4 +; SSE4-NEXT: movdqa %xmm5, %xmm0 +; SSE4-NEXT: blendvpd %xmm3, %xmm1 +; SSE4-NEXT: movapd %xmm4, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test174: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm4, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm4, %xmm4, %xmm4 +; AVX1-NEXT: vpxor %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vxorps %xmm3, %xmm1, %xmm5 +; AVX1-NEXT: vxorps %xmm3, %xmm0, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm5, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test174: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpxor %ymm2, %ymm1, %ymm3 +; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm2 +; AVX2-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3 +; AVX2-NEXT: vpxor %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test174: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxuq %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ule <4 x i64> %a, %b + %sel = select <4 x i1> %cmp, <4 x i64> %b, <4 x i64> %a + ret <4 x i64> %sel +} + +define <4 x i64> @test175(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: test175: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm3, %xmm4 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: movdqa %xmm1, %xmm6 +; SSE2-NEXT: pxor %xmm5, %xmm6 +; SSE2-NEXT: movdqa %xmm6, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm4 +; SSE2-NEXT: movdqa %xmm2, %xmm6 +; SSE2-NEXT: pxor %xmm5, %xmm6 +; SSE2-NEXT: pxor %xmm0, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm7 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm6, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm5 +; SSE2-NEXT: por %xmm2, %xmm5 +; SSE2-NEXT: pand %xmm4, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm4 +; SSE2-NEXT: por %xmm3, %xmm4 +; SSE2-NEXT: movdqa %xmm5, %xmm0 +; SSE2-NEXT: movdqa %xmm4, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test175: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm4 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm3, %xmm6 +; SSE4-NEXT: pxor %xmm0, %xmm6 +; SSE4-NEXT: movdqa %xmm1, %xmm5 +; SSE4-NEXT: pxor %xmm0, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm6, %xmm5 +; SSE4-NEXT: movdqa %xmm2, %xmm6 +; SSE4-NEXT: pxor %xmm0, %xmm6 +; SSE4-NEXT: pxor %xmm4, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm6, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm4 +; SSE4-NEXT: movdqa %xmm5, %xmm0 +; SSE4-NEXT: blendvpd %xmm3, %xmm1 +; SSE4-NEXT: movapd %xmm4, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test175: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm4, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vxorps %xmm3, %xmm1, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm0, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test175: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpxor %ymm2, %ymm1, %ymm3 +; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm2 +; AVX2-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test175: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminuq %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ugt <4 x i64> %a, %b + %sel = select <4 x i1> %cmp, <4 x i64> %b, <4 x i64> %a + ret <4 x i64> %sel +} + +define <4 x i64> @test176(<4 x i64> %a, <4 x i64> %b) { +; SSE2-LABEL: test176: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm4 +; SSE2-NEXT: pxor %xmm7, %xmm4 +; SSE2-NEXT: movdqa %xmm3, %xmm5 +; SSE2-NEXT: pxor %xmm7, %xmm5 +; SSE2-NEXT: movdqa %xmm5, %xmm6 +; SSE2-NEXT: pcmpgtd %xmm4, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3] +; SSE2-NEXT: pand %xmm8, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm8 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm4 +; SSE2-NEXT: movdqa %xmm8, %xmm9 +; SSE2-NEXT: pxor %xmm4, %xmm9 +; SSE2-NEXT: movdqa %xmm0, %xmm6 +; SSE2-NEXT: pxor %xmm7, %xmm6 +; SSE2-NEXT: pxor %xmm2, %xmm7 +; SSE2-NEXT: movdqa %xmm7, %xmm5 +; SSE2-NEXT: pcmpgtd %xmm6, %xmm5 +; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm5[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm6, %xmm7 +; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3] +; SSE2-NEXT: pand %xmm10, %xmm6 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3] +; SSE2-NEXT: por %xmm6, %xmm5 +; SSE2-NEXT: pxor %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm2, %xmm5 +; SSE2-NEXT: pandn %xmm0, %xmm4 +; SSE2-NEXT: por %xmm5, %xmm4 +; SSE2-NEXT: pandn %xmm3, %xmm8 +; SSE2-NEXT: pandn %xmm1, %xmm9 +; SSE2-NEXT: por %xmm8, %xmm9 +; SSE2-NEXT: movdqa %xmm4, %xmm0 +; SSE2-NEXT: movdqa %xmm9, %xmm1 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test176: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm4 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm1, %xmm6 +; SSE4-NEXT: pxor %xmm0, %xmm6 +; SSE4-NEXT: movdqa %xmm3, %xmm5 +; SSE4-NEXT: pxor %xmm0, %xmm5 +; SSE4-NEXT: pcmpgtq %xmm6, %xmm5 +; SSE4-NEXT: pcmpeqd %xmm6, %xmm6 +; SSE4-NEXT: pxor %xmm6, %xmm5 +; SSE4-NEXT: movdqa %xmm4, %xmm7 +; SSE4-NEXT: pxor %xmm0, %xmm7 +; SSE4-NEXT: pxor %xmm2, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm7, %xmm0 +; SSE4-NEXT: pxor %xmm6, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm4 +; SSE4-NEXT: movdqa %xmm5, %xmm0 +; SSE4-NEXT: blendvpd %xmm3, %xmm1 +; SSE4-NEXT: movapd %xmm4, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test176: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vxorps %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 +; AVX1-NEXT: vxorps %xmm3, %xmm4, %xmm4 +; AVX1-NEXT: vpcmpgtq %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm4, %xmm4, %xmm4 +; AVX1-NEXT: vpxor %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vxorps %xmm3, %xmm0, %xmm5 +; AVX1-NEXT: vxorps %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vpcmpgtq %xmm5, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test176: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm3 +; AVX2-NEXT: vpxor %ymm2, %ymm1, %ymm2 +; AVX2-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3 +; AVX2-NEXT: vpxor %ymm3, %ymm2, %ymm2 +; AVX2-NEXT: vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test176: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminuq %ymm1, %ymm0, %ymm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp uge <4 x i64> %a, %b + %sel = select <4 x i1> %cmp, <4 x i64> %b, <4 x i64> %a + ret <4 x i64> %sel +} + +define <2 x i64> @test177(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: test177: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test177: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm2 +; SSE4-NEXT: movdqa %xmm1, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm2, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm1 +; SSE4-NEXT: movapd %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test177: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm2 +; AVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test177: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm2 +; AVX2-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test177: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminsq %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp slt <2 x i64> %a, %b + %sel = select <2 x i1> %cmp, <2 x i64> %a, <2 x i64> %b + ret <2 x i64> %sel +} + +define <2 x i64> @test178(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: test178: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test178: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm2 +; SSE4-NEXT: pcmpgtq %xmm1, %xmm0 +; SSE4-NEXT: pcmpeqd %xmm3, %xmm3 +; SSE4-NEXT: pxor %xmm3, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm1 +; SSE4-NEXT: movapd %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test178: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test178: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm2 +; AVX2-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX2-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test178: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminsq %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sle <2 x i64> %a, %b + %sel = select <2 x i1> %cmp, <2 x i64> %a, <2 x i64> %b + ret <2 x i64> %sel +} + +define <2 x i64> @test179(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: test179: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test179: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm2 +; SSE4-NEXT: pcmpgtq %xmm1, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm1 +; SSE4-NEXT: movapd %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test179: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm2 +; AVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test179: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm2 +; AVX2-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test179: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxsq %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sgt <2 x i64> %a, %b + %sel = select <2 x i1> %cmp, <2 x i64> %a, <2 x i64> %b + ret <2 x i64> %sel +} + +define <2 x i64> @test180(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: test180: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test180: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm2 +; SSE4-NEXT: movdqa %xmm1, %xmm3 +; SSE4-NEXT: pcmpgtq %xmm2, %xmm3 +; SSE4-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE4-NEXT: pxor %xmm3, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm1 +; SSE4-NEXT: movapd %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test180: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test180: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm2 +; AVX2-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX2-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test180: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxsq %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sge <2 x i64> %a, %b + %sel = select <2 x i1> %cmp, <2 x i64> %a, <2 x i64> %b + ret <2 x i64> %sel +} + +define <2 x i64> @test181(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: test181: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test181: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm2 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm2, %xmm3 +; SSE4-NEXT: pxor %xmm0, %xmm3 +; SSE4-NEXT: pxor %xmm1, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm3, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm1 +; SSE4-NEXT: movapd %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test181: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm3 +; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test181: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm3 +; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm2 +; AVX2-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test181: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminuq %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ult <2 x i64> %a, %b + %sel = select <2 x i1> %cmp, <2 x i64> %a, <2 x i64> %b + ret <2 x i64> %sel +} + +define <2 x i64> @test182(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: test182: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test182: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm2 +; SSE4-NEXT: movdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm1, %xmm0 +; SSE4-NEXT: pxor %xmm3, %xmm0 +; SSE4-NEXT: pxor %xmm2, %xmm3 +; SSE4-NEXT: pcmpgtq %xmm0, %xmm3 +; SSE4-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE4-NEXT: pxor %xmm3, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm1 +; SSE4-NEXT: movapd %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test182: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm3 +; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test182: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm3 +; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm2 +; AVX2-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX2-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test182: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminuq %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ule <2 x i64> %a, %b + %sel = select <2 x i1> %cmp, <2 x i64> %a, <2 x i64> %b + ret <2 x i64> %sel +} + +define <2 x i64> @test183(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: test183: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pand %xmm3, %xmm0 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test183: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm2 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm1, %xmm3 +; SSE4-NEXT: pxor %xmm0, %xmm3 +; SSE4-NEXT: pxor %xmm2, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm3, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm1 +; SSE4-NEXT: movapd %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test183: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm3 +; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test183: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm3 +; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm2 +; AVX2-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test183: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxuq %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ugt <2 x i64> %a, %b + %sel = select <2 x i1> %cmp, <2 x i64> %a, <2 x i64> %b + ret <2 x i64> %sel +} + +define <2 x i64> @test184(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: test184: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm0, %xmm3 +; SSE2-NEXT: pandn %xmm1, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test184: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm2 +; SSE4-NEXT: movdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: pxor %xmm3, %xmm0 +; SSE4-NEXT: pxor %xmm1, %xmm3 +; SSE4-NEXT: pcmpgtq %xmm0, %xmm3 +; SSE4-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE4-NEXT: pxor %xmm3, %xmm0 +; SSE4-NEXT: blendvpd %xmm2, %xmm1 +; SSE4-NEXT: movapd %xmm1, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test184: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm3 +; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test184: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm3 +; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm2 +; AVX2-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX2-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test184: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxuq %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp uge <2 x i64> %a, %b + %sel = select <2 x i1> %cmp, <2 x i64> %a, <2 x i64> %b + ret <2 x i64> %sel +} + +define <2 x i64> @test185(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: test185: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm3 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test185: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm2 +; SSE4-NEXT: movdqa %xmm1, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm2, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm2 +; SSE4-NEXT: movapd %xmm2, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test185: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm2 +; AVX1-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test185: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm2 +; AVX2-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test185: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxsq %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp slt <2 x i64> %a, %b + %sel = select <2 x i1> %cmp, <2 x i64> %b, <2 x i64> %a + ret <2 x i64> %sel +} + +define <2 x i64> @test186(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: test186: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test186: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm2 +; SSE4-NEXT: pcmpgtq %xmm1, %xmm0 +; SSE4-NEXT: pcmpeqd %xmm3, %xmm3 +; SSE4-NEXT: pxor %xmm3, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm2 +; SSE4-NEXT: movapd %xmm2, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test186: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test186: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm2 +; AVX2-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX2-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test186: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxsq %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sle <2 x i64> %a, %b + %sel = select <2 x i1> %cmp, <2 x i64> %b, <2 x i64> %a + ret <2 x i64> %sel +} + +define <2 x i64> @test187(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: test187: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm3 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test187: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm2 +; SSE4-NEXT: pcmpgtq %xmm1, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm2 +; SSE4-NEXT: movapd %xmm2, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test187: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm2 +; AVX1-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test187: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm2 +; AVX2-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test187: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminsq %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sgt <2 x i64> %a, %b + %sel = select <2 x i1> %cmp, <2 x i64> %b, <2 x i64> %a + ret <2 x i64> %sel +} + +define <2 x i64> @test188(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: test188: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,0,2147483648,0] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test188: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm2 +; SSE4-NEXT: movdqa %xmm1, %xmm3 +; SSE4-NEXT: pcmpgtq %xmm2, %xmm3 +; SSE4-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE4-NEXT: pxor %xmm3, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm2 +; SSE4-NEXT: movapd %xmm2, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test188: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test188: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm2 +; AVX2-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX2-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test188: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminsq %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp sge <2 x i64> %a, %b + %sel = select <2 x i1> %cmp, <2 x i64> %b, <2 x i64> %a + ret <2 x i64> %sel +} + +define <2 x i64> @test189(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: test189: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm3 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test189: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm2 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm2, %xmm3 +; SSE4-NEXT: pxor %xmm0, %xmm3 +; SSE4-NEXT: pxor %xmm1, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm3, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm2 +; SSE4-NEXT: movapd %xmm2, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test189: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm3 +; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test189: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm3 +; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm2 +; AVX2-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test189: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxuq %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ult <2 x i64> %a, %b + %sel = select <2 x i1> %cmp, <2 x i64> %b, <2 x i64> %a + ret <2 x i64> %sel +} + +define <2 x i64> @test190(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: test190: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test190: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm2 +; SSE4-NEXT: movdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm1, %xmm0 +; SSE4-NEXT: pxor %xmm3, %xmm0 +; SSE4-NEXT: pxor %xmm2, %xmm3 +; SSE4-NEXT: pcmpgtq %xmm0, %xmm3 +; SSE4-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE4-NEXT: pxor %xmm3, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm2 +; SSE4-NEXT: movapd %xmm2, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test190: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm3 +; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test190: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm3 +; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm2 +; AVX2-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX2-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test190: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpmaxuq %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ule <2 x i64> %a, %b + %sel = select <2 x i1> %cmp, <2 x i64> %b, <2 x i64> %a + ret <2 x i64> %sel +} + +define <2 x i64> @test191(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: test191: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm1, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm3 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: pand %xmm2, %xmm1 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test191: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm2 +; SSE4-NEXT: movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: movdqa %xmm1, %xmm3 +; SSE4-NEXT: pxor %xmm0, %xmm3 +; SSE4-NEXT: pxor %xmm2, %xmm0 +; SSE4-NEXT: pcmpgtq %xmm3, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm2 +; SSE4-NEXT: movapd %xmm2, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test191: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm3 +; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test191: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm3 +; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm2 +; AVX2-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test191: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminuq %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp ugt <2 x i64> %a, %b + %sel = select <2 x i1> %cmp, <2 x i64> %b, <2 x i64> %a + ret <2 x i64> %sel +} + +define <2 x i64> @test192(<2 x i64> %a, <2 x i64> %b) { +; SSE2-LABEL: test192: +; SSE2: # BB#0: # %entry +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm2, %xmm3 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm4 +; SSE2-NEXT: pcmpgtd %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2] +; SSE2-NEXT: pcmpeqd %xmm3, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3] +; SSE2-NEXT: pand %xmm5, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3] +; SSE2-NEXT: por %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: pxor %xmm3, %xmm2 +; SSE2-NEXT: pandn %xmm1, %xmm3 +; SSE2-NEXT: pandn %xmm0, %xmm2 +; SSE2-NEXT: por %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE4-LABEL: test192: +; SSE4: # BB#0: # %entry +; SSE4-NEXT: movdqa %xmm0, %xmm2 +; SSE4-NEXT: movdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808] +; SSE4-NEXT: pxor %xmm3, %xmm0 +; SSE4-NEXT: pxor %xmm1, %xmm3 +; SSE4-NEXT: pcmpgtq %xmm0, %xmm3 +; SSE4-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE4-NEXT: pxor %xmm3, %xmm0 +; SSE4-NEXT: blendvpd %xmm1, %xmm2 +; SSE4-NEXT: movapd %xmm2, %xmm0 +; SSE4-NEXT: retq +; +; AVX1-LABEL: test192: +; AVX1: # BB#0: # %entry +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm3 +; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm2 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: test192: +; AVX2: # BB#0: # %entry +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm3 +; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm2 +; AVX2-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3 +; AVX2-NEXT: vpxor %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; AVX512BW-LABEL: test192: +; AVX512BW: # BB#0: # %entry +; AVX512BW-NEXT: vpminuq %xmm1, %xmm0, %xmm0 +; AVX512BW-NEXT: retq +entry: + %cmp = icmp uge <2 x i64> %a, %b + %sel = select <2 x i1> %cmp, <2 x i64> %b, <2 x i64> %a + ret <2 x i64> %sel } diff --git a/test/CodeGen/X86/vselect.ll b/test/CodeGen/X86/vselect.ll index 8c8092888834..359ea7eb3ee5 100644 --- a/test/CodeGen/X86/vselect.ll +++ b/test/CodeGen/X86/vselect.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+sse2,-sse4.1 < %s | FileCheck %s ; Verify that we don't emit packed vector shifts instructions if the diff --git a/test/CodeGen/X86/vshift_scalar.ll b/test/CodeGen/X86/vshift_scalar.ll index 9dd8478caaed..87eec3f9e97b 100644 --- a/test/CodeGen/X86/vshift_scalar.ll +++ b/test/CodeGen/X86/vshift_scalar.ll @@ -1,4 +1,5 @@ ; RUN: llc < %s +; REQUIRES: default_triple ; Legalization test that requires scalarizing a vector. diff --git a/test/CodeGen/X86/wide-integer-cmp.ll b/test/CodeGen/X86/wide-integer-cmp.ll new file mode 100644 index 000000000000..c45a0541e6a7 --- /dev/null +++ b/test/CodeGen/X86/wide-integer-cmp.ll @@ -0,0 +1,130 @@ +; RUN: llc -mtriple=i686-linux-gnu %s -o - | FileCheck %s + + +define i32 @branch_eq(i64 %a, i64 %b) { +entry: + %cmp = icmp eq i64 %a, %b + br i1 %cmp, label %bb1, label %bb2 +bb1: + ret i32 1 +bb2: + ret i32 2 + +; CHECK-LABEL: branch_eq: +; CHECK: movl 4(%esp), [[LHSLo:%[a-z]+]] +; CHECK: movl 8(%esp), [[LHSHi:%[a-z]+]] +; CHECK: xorl 16(%esp), [[LHSHi]] +; CHECK: xorl 12(%esp), [[LHSLo]] +; CHECK: orl [[LHSHi]], [[LHSLo]] +; CHECK: jne [[FALSE:.LBB[0-9_]+]] +; CHECK: movl $1, %eax +; CHECK: retl +; CHECK: [[FALSE]]: +; CHECK: movl $2, %eax +; CHECK: retl +} + +define i32 @branch_slt(i64 %a, i64 %b) { +entry: + %cmp = icmp slt i64 %a, %b + br i1 %cmp, label %bb1, label %bb2 +bb1: + ret i32 1 +bb2: + ret i32 2 + +; CHECK-LABEL: branch_slt: +; CHECK: movl 4(%esp), [[LHSLo:%[a-z]+]] +; CHECK: movl 8(%esp), [[LHSHi:%[a-z]+]] +; CHECK: cmpl 12(%esp), [[LHSLo]] +; CHECK: sbbl 16(%esp), [[LHSHi]] +; CHECK: jge [[FALSE:.LBB[0-9_]+]] +; CHECK: movl $1, %eax +; CHECK: retl +; CHECK: [[FALSE]]: +; CHECK: movl $2, %eax +; CHECK: retl +} + +define i32 @branch_ule(i64 %a, i64 %b) { +entry: + %cmp = icmp ule i64 %a, %b + br i1 %cmp, label %bb1, label %bb2 +bb1: + ret i32 1 +bb2: + ret i32 2 + +; CHECK-LABEL: branch_ule: +; CHECK: movl 12(%esp), [[RHSLo:%[a-z]+]] +; CHECK: movl 16(%esp), [[RHSHi:%[a-z]+]] +; CHECK: cmpl 4(%esp), [[RHSLo]] +; CHECK: sbbl 8(%esp), [[RHSHi]] +; CHECK: jb [[FALSE:.LBB[0-9_]+]] +; CHECK: movl $1, %eax +; CHECK: retl +; CHECK: [[FALSE]]: +; CHECK: movl $2, %eax +; CHECK: retl +} + +define i32 @set_gt(i64 %a, i64 %b) { +entry: + %cmp = icmp sgt i64 %a, %b + %res = select i1 %cmp, i32 1, i32 0 + ret i32 %res + +; CHECK-LABEL: set_gt: +; CHECK: movl 12(%esp), [[RHSLo:%[a-z]+]] +; CHECK: movl 16(%esp), [[RHSHi:%[a-z]+]] +; CHECK: cmpl 4(%esp), [[RHSLo]] +; CHECK: sbbl 8(%esp), [[RHSHi]] +; CHECK: setl %al +; CHECK: retl +} + +define i32 @test_wide(i128 %a, i128 %b) { +entry: + %cmp = icmp slt i128 %a, %b + br i1 %cmp, label %bb1, label %bb2 +bb1: + ret i32 1 +bb2: + ret i32 2 + +; CHECK-LABEL: test_wide: +; CHECK: cmpl 24(%esp) +; CHECK: sbbl 28(%esp) +; CHECK: sbbl 32(%esp) +; CHECK: sbbl 36(%esp) +; CHECK: jge [[FALSE:.LBB[0-9_]+]] +; CHECK: movl $1, %eax +; CHECK: retl +; CHECK: [[FALSE]]: +; CHECK: movl $2, %eax +; CHECK: retl +} + +define i32 @test_carry_false(i64 %a, i64 %b) { +entry: + %x = and i64 %a, -4294967296 ;0xffffffff00000000 + %y = and i64 %b, -4294967296 + %cmp = icmp slt i64 %x, %y + br i1 %cmp, label %bb1, label %bb2 +bb1: + ret i32 1 +bb2: + ret i32 2 + +; The comparison of the low bits will be folded to a CARRY_FALSE node. Make +; sure the code can handle that. +; CHECK-LABEL: carry_false: +; CHECK: movl 8(%esp), [[LHSHi:%[a-z]+]] +; CHECK: cmpl 16(%esp), [[LHSHi]] +; CHECK: jge [[FALSE:.LBB[0-9_]+]] +; CHECK: movl $1, %eax +; CHECK: retl +; CHECK: [[FALSE]]: +; CHECK: movl $2, %eax +; CHECK: retl +} diff --git a/test/CodeGen/X86/widen_load-2.ll b/test/CodeGen/X86/widen_load-2.ll index 6f1bd7541231..fad1fa32559a 100644 --- a/test/CodeGen/X86/widen_load-2.ll +++ b/test/CodeGen/X86/widen_load-2.ll @@ -191,9 +191,7 @@ define void @rot(%i8vec3pack* nocapture sret %result, %i8vec3pack* %X, %i8vec3pa ; CHECK-NEXT: movd %[[CONSTANT1]], %e[[R1:[abcd]]]x ; CHECK-NEXT: movw %[[R1]]x, (%[[PTR1:.*]]) ; CHECK-NEXT: movb $1, 2(%[[PTR1]]) -; CHECK-NEXT: movl (%[[PTR0]]), [[TMP1:%e[abcd]+x]] -; CHECK-NEXT: movl [[TMP1]], [[TMP2:.*]] -; CHECK-NEXT: pmovzxbd [[TMP2]], %[[X0:xmm[0-9]+]] +; CHECK-NEXT: pmovzxbd (%[[PTR0]]), %[[X0:xmm[0-9]+]] ; CHECK-NEXT: movdqa %[[X0]], %[[X1:xmm[0-9]+]] ; CHECK-NEXT: psrld $1, %[[X1]] ; CHECK-NEXT: pblendw $192, %[[X0]], %[[X1]] diff --git a/test/CodeGen/X86/widen_shuffle-1.ll b/test/CodeGen/X86/widen_shuffle-1.ll index 302805213d06..66ba6350c8a8 100644 --- a/test/CodeGen/X86/widen_shuffle-1.ll +++ b/test/CodeGen/X86/widen_shuffle-1.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -march=x86 -mattr=+sse4.2 | FileCheck %s target triple = "x86_64-unknown-unknown" diff --git a/test/CodeGen/X86/win-catchpad-csrs.ll b/test/CodeGen/X86/win-catchpad-csrs.ll new file mode 100644 index 000000000000..327ee45b4326 --- /dev/null +++ b/test/CodeGen/X86/win-catchpad-csrs.ll @@ -0,0 +1,268 @@ +; RUN: llc -verify-machineinstrs -mtriple=i686-pc-windows-msvc < %s | FileCheck --check-prefix=X86 %s +; RUN: llc -verify-machineinstrs -mtriple=x86_64-pc-windows-msvc < %s | FileCheck --check-prefix=X64 %s + +%rtti.TypeDescriptor2 = type { i8**, i8*, [3 x i8] } +%eh.CatchableType = type { i32, i8*, i32, i32, i32, i32, i8* } +%eh.CatchableTypeArray.1 = type { i32, [1 x %eh.CatchableType*] } +%eh.ThrowInfo = type { i32, i8*, i8*, i8* } + +$"\01??_R0H@8" = comdat any + +@"\01??_7type_info@@6B@" = external constant i8* +@"\01??_R0H@8" = linkonce_odr global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".H\00" }, comdat + +declare i32 @getint() +declare void @useints(...) +declare void @f(i32 %p) +declare i32 @__CxxFrameHandler3(...) + +define i32 @try_catch_catch() personality i32 (...)* @__CxxFrameHandler3 { +entry: + %a = call i32 @getint() + %b = call i32 @getint() + %c = call i32 @getint() + %d = call i32 @getint() + call void (...) @useints(i32 %a, i32 %b, i32 %c, i32 %d) + invoke void @f(i32 1) + to label %try.cont unwind label %catch.dispatch + +try.cont: + ret i32 0 + +catch.dispatch: + %cs = catchswitch within none [label %handler1] unwind to caller + +handler1: + %h1 = catchpad within %cs [%rtti.TypeDescriptor2* @"\01??_R0H@8", i32 0, i8* null] + call void @f(i32 2) [ "funclet"(token %h1) ] + catchret from %h1 to label %try.cont +} + +; X86-LABEL: _try_catch_catch: +; X86: pushl %ebp +; X86: movl %esp, %ebp +; X86: pushl %ebx +; X86: pushl %edi +; X86: pushl %esi +; X86: subl ${{[0-9]+}}, %esp +; X86: calll _getint +; X86: calll _getint +; X86: calll _getint +; X86: calll _getint +; X86: calll _useints +; X86: movl $0, -{{[0-9]+}}(%ebp) +; X86: movl $1, (%esp) +; X86: calll _f +; X86: [[contbb:LBB0_[0-9]+]]: # %try.cont +; X86: popl %esi +; X86: popl %edi +; X86: popl %ebx +; X86: popl %ebp +; X86: retl + +; X86: [[restorebb:LBB0_[0-9]+]]: +; X86: addl $12, %ebp +; X86: jmp [[contbb]] + +; X86: "?catch$[[catch1bb:[0-9]+]]@?0?try_catch_catch@4HA": +; X86: LBB0_[[catch1bb]]: # %handler1{{$}} +; X86: pushl %ebp +; X86-NOT: pushl +; X86: subl $16, %esp +; X86: addl $12, %ebp +; X86: movl $1, -{{[0-9]+}}(%ebp) +; X86: movl $2, (%esp) +; X86: calll _f +; X86: movl $[[restorebb]], %eax +; X86-NEXT: addl $16, %esp +; X86-NEXT: popl %ebp +; X86-NEXT: retl + +; X86: L__ehtable$try_catch_catch: +; X86: $handlerMap$0$try_catch_catch: +; X86: .long 0 +; X86: .long "??_R0H@8" +; X86: .long 0 +; X86: .long "?catch$[[catch1bb]]@?0?try_catch_catch@4HA" + +; X64-LABEL: try_catch_catch: +; X64: pushq %rbp +; X64: .seh_pushreg 5 +; X64: pushq %rsi +; X64: .seh_pushreg 6 +; X64: pushq %rdi +; X64: .seh_pushreg 7 +; X64: pushq %rbx +; X64: .seh_pushreg 3 +; X64: subq $40, %rsp +; X64: .seh_stackalloc 40 +; X64: leaq 32(%rsp), %rbp +; X64: .seh_setframe 5, 32 +; X64: .seh_endprologue +; X64: movq $-2, (%rbp) +; X64: callq getint +; X64: callq getint +; X64: callq getint +; X64: callq getint +; X64: callq useints +; X64: movl $1, %ecx +; X64: callq f +; X64: [[contbb:\.LBB0_[0-9]+]]: # Block address taken +; X64-NEXT: # %try.cont +; X64: addq $40, %rsp +; X64: popq %rbp +; X64: retq + +; X64: "?catch$[[catch1bb:[0-9]+]]@?0?try_catch_catch@4HA": +; X64: LBB0_[[catch1bb]]: # %handler1{{$}} +; X64: movq %rdx, 16(%rsp) +; X64: pushq %rbp +; X64: .seh_pushreg 5 +; X64: pushq %rsi +; X64: .seh_pushreg 6 +; X64: pushq %rdi +; X64: .seh_pushreg 7 +; X64: pushq %rbx +; X64: .seh_pushreg 3 +; X64: subq $40, %rsp +; X64: .seh_stackalloc 40 +; X64: leaq 32(%rdx), %rbp +; X64: .seh_endprologue +; X64: movl $2, %ecx +; X64: callq f +; X64: leaq [[contbb]](%rip), %rax +; X64: addq $40, %rsp +; X64: popq %rbx +; X64: popq %rdi +; X64: popq %rsi +; X64: popq %rbp +; X64: retq + +; X64: $handlerMap$0$try_catch_catch: +; X64: .long 0 +; X64: .long "??_R0H@8"@IMGREL +; X64: .long 0 +; X64: .long "?catch$[[catch1bb]]@?0?try_catch_catch@4HA"@IMGREL +; X64: .long 88 + +define i32 @try_one_csr() personality i32 (...)* @__CxxFrameHandler3 { +entry: + %a = call i32 @getint() + %b = call i32 @getint() + call void (...) @useints(i32 %a) + invoke void @f(i32 1) + to label %try.cont unwind label %catch.dispatch + +catch.dispatch: + %cs = catchswitch within none [label %handler1] unwind to caller + +handler1: + %0 = catchpad within %cs [%rtti.TypeDescriptor2* @"\01??_R0H@8", i32 0, i8* null] + catchret from %0 to label %try.cont + +try.cont: + ret i32 0 +} + +; X64-LABEL: try_one_csr: +; X64: pushq %rbp +; X64: .seh_pushreg 5 +; X64: pushq %rsi +; X64: .seh_pushreg 6 +; X64-NOT: pushq +; X64: subq $40, %rsp +; X64: .seh_stackalloc 40 +; X64: leaq 32(%rsp), %rbp +; X64: .seh_setframe 5, 32 +; X64: .seh_endprologue +; X64: callq getint +; X64: callq getint +; X64: callq useints +; X64: movl $1, %ecx +; X64: callq f +; X64: [[contbb:\.LBB1_[0-9]+]]: # Block address taken +; X64-NEXT: # %try.cont +; X64: addq $40, %rsp +; X64-NOT: popq +; X64: popq %rsi +; X64: popq %rbp +; X64: retq + +; X64: "?catch$[[catch1bb:[0-9]+]]@?0?try_one_csr@4HA": +; X64: LBB1_[[catch1bb]]: # %handler1{{$}} +; X64: movq %rdx, 16(%rsp) +; X64: pushq %rbp +; X64: .seh_pushreg 5 +; X64: pushq %rsi +; X64: .seh_pushreg 6 +; X64: subq $40, %rsp +; X64: .seh_stackalloc 40 +; X64: leaq 32(%rdx), %rbp +; X64: .seh_endprologue +; X64: leaq [[contbb]](%rip), %rax +; X64: addq $40, %rsp +; X64: popq %rsi +; X64: popq %rbp +; X64: retq + +; X64: $handlerMap$0$try_one_csr: +; X64: .long 0 +; X64: .long "??_R0H@8"@IMGREL +; X64: .long 0 +; X64: .long "?catch$[[catch1bb]]@?0?try_one_csr@4HA"@IMGREL +; X64: .long 72 + +define i32 @try_no_csr() personality i32 (...)* @__CxxFrameHandler3 { +entry: + invoke void @f(i32 1) + to label %try.cont unwind label %catch.dispatch + +catch.dispatch: + %cs = catchswitch within none [label %handler1] unwind to caller + +handler1: + %cp1 = catchpad within %cs [%rtti.TypeDescriptor2* @"\01??_R0H@8", i32 0, i8* null] + catchret from %cp1 to label %try.cont + +try.cont: + ret i32 0 +} + +; X64-LABEL: try_no_csr: +; X64: pushq %rbp +; X64: .seh_pushreg 5 +; X64-NOT: pushq +; X64: subq $48, %rsp +; X64: .seh_stackalloc 48 +; X64: leaq 48(%rsp), %rbp +; X64: .seh_setframe 5, 48 +; X64: .seh_endprologue +; X64: movl $1, %ecx +; X64: callq f +; X64: [[contbb:\.LBB2_[0-9]+]]: # Block address taken +; X64-NEXT: # %try.cont +; X64: addq $48, %rsp +; X64-NOT: popq +; X64: popq %rbp +; X64: retq + +; X64: "?catch$[[catch1bb:[0-9]+]]@?0?try_no_csr@4HA": +; X64: LBB2_[[catch1bb]]: # %handler1{{$}} +; X64: movq %rdx, 16(%rsp) +; X64: pushq %rbp +; X64: .seh_pushreg 5 +; X64: subq $32, %rsp +; X64: .seh_stackalloc 32 +; X64: leaq 48(%rdx), %rbp +; X64: .seh_endprologue +; X64: leaq [[contbb]](%rip), %rax +; X64: addq $32, %rsp +; X64: popq %rbp +; X64: retq + +; X64: $handlerMap$0$try_no_csr: +; X64: .long 0 +; X64: .long "??_R0H@8"@IMGREL +; X64: .long 0 +; X64: .long "?catch$[[catch1bb]]@?0?try_no_csr@4HA"@IMGREL +; X64: .long 56 diff --git a/test/CodeGen/X86/win-catchpad-nested-cxx.ll b/test/CodeGen/X86/win-catchpad-nested-cxx.ll new file mode 100644 index 000000000000..ac4598385cd1 --- /dev/null +++ b/test/CodeGen/X86/win-catchpad-nested-cxx.ll @@ -0,0 +1,105 @@ +; RUN: llc -verify-machineinstrs -mtriple=i686-pc-windows-msvc < %s \ +; RUN: | FileCheck --check-prefix=CHECK --check-prefix=X86 %s +; RUN: llc -verify-machineinstrs -mtriple=x86_64-pc-windows-msvc < %s \ +; RUN: | FileCheck --check-prefix=CHECK --check-prefix=X64 %s + +; Loosely based on IR for this C++ source code: +; void f(int p); +; void try_in_catch() { +; try { +; f(1); +; } catch (...) { +; try { +; f(2); +; } catch (...) { +; f(3); +; } +; } +; } + +declare void @f(i32 %p) +declare i32 @__CxxFrameHandler3(...) + +define i32 @try_in_catch() personality i32 (...)* @__CxxFrameHandler3 { +entry: + invoke void @f(i32 1) + to label %try.cont unwind label %catch.dispatch.1 +try.cont: + ret i32 0 + +catch.dispatch.1: + %cs1 = catchswitch within none [label %handler1] unwind to caller +handler1: + %h1 = catchpad within %cs1 [i8* null, i32 64, i8* null] + invoke void @f(i32 2) [ "funclet"(token %h1) ] + to label %catchret1 unwind label %catch.dispatch.2 +catchret1: + catchret from %h1 to label %try.cont + +catch.dispatch.2: + %cs2 = catchswitch within %h1 [label %handler2] unwind to caller +handler2: + %h2 = catchpad within %cs2 [i8* null, i32 64, i8* null] + call void @f(i32 3) + catchret from %h2 to label %catchret1 +} + +; X86-LABEL: L__ehtable$try_in_catch: +; X64-LABEL: $cppxdata$try_in_catch: +; CHECK-NEXT: .long 429065506 +; CHECK-NEXT: .long 4 +; CHECK-NEXT: .long ($stateUnwindMap$try_in_catch) +; CHECK-NEXT: .long 2 +; CHECK-NEXT: .long ($tryMap$try_in_catch) +; ip2state num + ptr +; X86-NEXT: .long 0 +; X86-NEXT: .long 0 +; X64-NEXT: .long 7 +; X64-NEXT: .long ($ip2state$try_in_catch) +; unwindhelp offset +; X64-NEXT: .long 40 +; CHECK-NEXT: .long 0 +; EHFlags +; CHECK-NEXT: .long 1 + +; CHECK: $tryMap$try_in_catch: +; CHECK-NEXT: .long 2 +; CHECK-NEXT: .long 2 +; CHECK-NEXT: .long 3 +; CHECK-NEXT: .long 1 +; CHECK-NEXT: .long ($handlerMap$0$try_in_catch) +; CHECK-NEXT: .long 0 +; CHECK-NEXT: .long 0 +; CHECK-NEXT: .long 3 +; CHECK-NEXT: .long 1 +; CHECK-NEXT: .long ($handlerMap$1$try_in_catch) + +; CHECK: $handlerMap$0$try_in_catch: +; CHECK-NEXT: .long 64 +; CHECK-NEXT: .long 0 +; CHECK-NEXT: .long 0 +; CHECK-NEXT: .long "?catch${{[0-9]+}}@?0?try_in_catch@4HA" +; X64-NEXT: .long 56 + +; CHECK: $handlerMap$1$try_in_catch: +; CHECK-NEXT: .long 64 +; CHECK-NEXT: .long 0 +; CHECK-NEXT: .long 0 +; CHECK-NEXT: .long "?catch${{[0-9]+}}@?0?try_in_catch@4HA" +; X64-NEXT: .long 56 + +; X64: $ip2state$try_in_catch: +; X64-NEXT: .long .Lfunc_begin0@IMGREL +; X64-NEXT: .long -1 +; X64-NEXT: .long .Ltmp0@IMGREL+1 +; X64-NEXT: .long 0 +; X64-NEXT: .long .Ltmp1@IMGREL+1 +; X64-NEXT: .long -1 +; X64-NEXT: .long "?catch$2@?0?try_in_catch@4HA"@IMGREL +; X64-NEXT: .long 1 +; X64-NEXT: .long .Ltmp2@IMGREL+1 +; X64-NEXT: .long 2 +; X64-NEXT: .long .Ltmp3@IMGREL+1 +; X64-NEXT: .long 1 +; X64-NEXT: .long "?catch$4@?0?try_in_catch@4HA"@IMGREL +; X64-NEXT: .long 3 diff --git a/test/CodeGen/X86/win-catchpad-nested.ll b/test/CodeGen/X86/win-catchpad-nested.ll new file mode 100644 index 000000000000..7afcd9cc1f3e --- /dev/null +++ b/test/CodeGen/X86/win-catchpad-nested.ll @@ -0,0 +1,42 @@ +; RUN: llc -mtriple=x86_64-pc-windows-coreclr < %s | FileCheck %s + +declare void @ProcessCLRException() + +declare void @f() + +define void @test1() personality void ()* @ProcessCLRException { +entry: + invoke void @f() + to label %exit unwind label %catch.dispatch.1 +exit: + ret void + +catch.dispatch.1: + %cs1 = catchswitch within none [label %outer.catch] unwind to caller + +outer.catch: + %cp1 = catchpad within %cs1 [i32 1] + invoke void @f() [ "funclet"(token %cp1) ] + to label %outer.ret unwind label %catch.dispatch.2 +outer.ret: + catchret from %cp1 to label %exit + +catch.dispatch.2: + %cs2 = catchswitch within %cp1 [label %inner.catch] unwind to caller +inner.catch: + %cp2 = catchpad within %cs2 [i32 2] + catchret from %cp2 to label %outer.ret +} + +; Check the catchret targets +; CHECK-LABEL: test1: # @test1 +; CHECK: [[Exit:^[^: ]+]]: # Block address taken +; CHECK-NEXT: # %exit +; CHECK: [[OuterRet:^[^: ]+]]: # Block address taken +; CHECK-NEXT: # %outer.ret +; CHECK-NEXT: leaq [[Exit]](%rip), %rax +; CHECK: retq # CATCHRET +; CHECK: {{^[^: ]+}}: # %inner.catch +; CHECK: .seh_endprolog +; CHECK-NEXT: leaq [[OuterRet]](%rip), %rax +; CHECK: retq # CATCHRET diff --git a/test/CodeGen/X86/win-catchpad-varargs.ll b/test/CodeGen/X86/win-catchpad-varargs.ll new file mode 100644 index 000000000000..6508f3bd7d64 --- /dev/null +++ b/test/CodeGen/X86/win-catchpad-varargs.ll @@ -0,0 +1,101 @@ +; RUN: llc -mtriple=x86_64-windows-msvc < %s | FileCheck %s --check-prefix=X64 +; RUN: llc -mtriple=i686-windows-msvc < %s | FileCheck %s --check-prefix=X86 + +declare void @llvm.va_start(i8*) +declare void @llvm.va_end(i8*) +declare i32 @__CxxFrameHandler3(...) +declare void @g() + +define i32 @f(i32 %a, ...) personality i32 (...)* @__CxxFrameHandler3 { +entry: + %ap = alloca i8* + invoke void @g() + to label %return unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %cs1 = catchswitch within none [label %catch] unwind to caller + +catch: ; preds = %catch.dispatch + %0 = catchpad within %cs1 [i8* null, i32 64, i8* null] + %ap1 = bitcast i8** %ap to i8* + call void @llvm.va_start(i8* %ap1) + %argp.cur = load i8*, i8** %ap + %1 = bitcast i8* %argp.cur to i32* + %arg2 = load i32, i32* %1 + call void @llvm.va_end(i8* %ap1) + catchret from %0 to label %return + +return: ; preds = %entry, %catch + %retval.0 = phi i32 [ %arg2, %catch ], [ -1, %entry ] + ret i32 %retval.0 +} + +; X64-LABEL: .seh_proc f +; X64: pushq %rbp +; X64: subq $64, %rsp +; X64: leaq 64(%rsp), %rbp +; X64: movq $-2, -8(%rbp) +; X64: movl $-1, -20(%rbp) # 4-byte Folded Spill +; X64: callq g +; X64: .LBB0_1 +; X64: movl -20(%rbp), %eax # 4-byte Reload +; X64: addq $64, %rsp +; X64: popq %rbp + +; X64-LABEL: "?catch${{[0-9]}}@?0?f@4HA": +; X64: .seh_proc "?catch${{[0-9]}}@?0?f@4HA" +; X64: movq %rdx, 16(%rsp) +; X64: pushq %rbp +; X64: subq $32, %rsp +; X64: leaq 64(%rdx), %rbp +; arg2 is at RBP+40: +; start at arg2 +; + 8 for arg1 +; + 8 for retaddr +; + 8 for RBP +; + 64 for stackalloc +; - 64 for setframe +; = 40 +; X64: movl 24(%rbp), %eax +; X64: movl %eax, -20(%rbp) # 4-byte Spill +; X64: leaq .LBB0_1(%rip), %rax +; X64: addq $32, %rsp +; X64: popq %rbp +; X64: retq # CATCHRET + +; X86-LABEL: _f: # @f +; X86: pushl %ebp +; X86: movl %esp, %ebp +; X86: pushl %ebx +; X86: pushl %edi +; X86: pushl %esi +; X86: subl $24, %esp +; X86: movl $-1, -36(%ebp) +; X86: calll _g +; X86: LBB0_[[retbb:[0-9]+]]: +; X86: movl -36(%ebp), %eax +; X86: addl $24, %esp +; X86: popl %esi +; X86: popl %edi +; X86: popl %ebx +; X86: popl %ebp +; X86: retl + +; X86: LBB0_[[restorebb:[0-9]+]]: # Block address taken +; X86: addl $12, %ebp +; arg2 is at EBP offset 12: +; + 4 for arg1 +; + 4 for retaddr +; + 4 for EBP +; X86: movl 12(%ebp), %eax +; X86: movl %eax, -36(%ebp) +; X86: jmp LBB0_[[retbb]] + +; X86-LABEL: "?catch${{[0-9]}}@?0?f@4HA": +; X86: pushl %ebp +; X86: addl $12, %ebp +; Done due to mov %esp, %ebp +; X86: leal 12(%ebp), %eax +; X86: movl $LBB0_[[restorebb]], %eax +; X86: popl %ebp +; X86: retl # CATCHRET diff --git a/test/CodeGen/X86/win-catchpad.ll b/test/CodeGen/X86/win-catchpad.ll new file mode 100644 index 000000000000..836c53bda8e6 --- /dev/null +++ b/test/CodeGen/X86/win-catchpad.ll @@ -0,0 +1,353 @@ +; RUN: llc -verify-machineinstrs -mtriple=i686-pc-windows-msvc < %s | FileCheck --check-prefix=X86 %s +; RUN: llc -verify-machineinstrs -mtriple=x86_64-pc-windows-msvc < %s | FileCheck --check-prefix=X64 %s + +; Loosely based on IR for this C++ source code: +; void f(int p); +; int main() { +; try { +; f(1); +; } catch (int e) { +; f(e); +; } catch (...) { +; f(3); +; } +; } + +%rtti.TypeDescriptor2 = type { i8**, i8*, [3 x i8] } +%eh.CatchableType = type { i32, i8*, i32, i32, i32, i32, i8* } +%eh.CatchableTypeArray.1 = type { i32, [1 x %eh.CatchableType*] } +%eh.ThrowInfo = type { i32, i8*, i8*, i8* } + +$"\01??_R0H@8" = comdat any + +@"\01??_7type_info@@6B@" = external constant i8* +@"\01??_R0H@8" = linkonce_odr global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".H\00" }, comdat + + +declare void @f(i32 %p, i32* %l) +declare i1 @getbool() +declare i32 @__CxxFrameHandler3(...) + +define i32 @try_catch_catch() personality i32 (...)* @__CxxFrameHandler3 { +entry: + %e.addr = alloca i32 + %local = alloca i32 + invoke void @f(i32 1, i32* %local) + to label %try.cont unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %cs = catchswitch within none [label %handler1, label %handler2] unwind to caller + +handler1: + %h1 = catchpad within %cs [%rtti.TypeDescriptor2* @"\01??_R0H@8", i32 0, i32* %e.addr] + %e = load i32, i32* %e.addr + call void @f(i32 %e, i32* %local) [ "funclet"(token %h1) ] + catchret from %h1 to label %try.cont + +handler2: + %h2 = catchpad within %cs [i8* null, i32 64, i8* null] + call void @f(i32 3, i32* %local) [ "funclet"(token %h2) ] + catchret from %h2 to label %try.cont + +try.cont: + ret i32 0 +} + +; X86-LABEL: _try_catch_catch: +; X86: movl %esp, -[[sp_offset:[0-9]+]](%ebp) +; X86: movl $0, -{{[0-9]+}}(%ebp) +; X86: leal -[[local_offs:[0-9]+]](%ebp), %[[addr_reg:[a-z]+]] +; X86-DAG: movl %[[addr_reg]], 4(%esp) +; X86-DAG: movl $1, (%esp) +; X86: calll _f +; X86: [[contbb:LBB0_[0-9]+]]: # %try.cont +; X86: retl + +; X86: [[restorebb1:LBB0_[0-9]+]]: # Block address taken +; X86-NEXT: # %handler1 +; X86-NEXT: addl $12, %ebp +; X86: jmp [[contbb]] + +; FIXME: These should be de-duplicated. +; X86: [[restorebb2:LBB0_[0-9]+]]: # Block address taken +; X86-NEXT: # %handler2 +; X86-NEXT: addl $12, %ebp +; X86: jmp [[contbb]] + +; X86: "?catch$[[catch1bb:[0-9]+]]@?0?try_catch_catch@4HA": +; X86: LBB0_[[catch1bb]]: # %handler1{{$}} +; X86: pushl %ebp +; X86: subl $8, %esp +; X86: addl $12, %ebp +; X86: movl %esp, -[[sp_offset]](%ebp) +; X86-DAG: movl -32(%ebp), %[[e_reg:[a-z]+]] +; X86-DAG: leal -[[local_offs]](%ebp), %[[addr_reg:[a-z]+]] +; X86-DAG: movl $1, -{{[0-9]+}}(%ebp) +; X86-DAG: movl %[[addr_reg]], 4(%esp) +; X86-DAG: movl %[[e_reg]], (%esp) +; X86: calll _f +; X86-NEXT: movl $[[restorebb1]], %eax +; X86-NEXT: addl $8, %esp +; X86-NEXT: popl %ebp +; X86-NEXT: retl + +; X86: "?catch$[[catch2bb:[0-9]+]]@?0?try_catch_catch@4HA": +; X86: LBB0_[[catch2bb]]: # %handler2{{$}} +; X86: pushl %ebp +; X86: subl $8, %esp +; X86: addl $12, %ebp +; X86: movl %esp, -[[sp_offset]](%ebp) +; X86-DAG: leal -[[local_offs]](%ebp), %[[addr_reg:[a-z]+]] +; X86-DAG: movl $1, -{{[0-9]+}}(%ebp) +; X86-DAG: movl %[[addr_reg]], 4(%esp) +; X86-DAG: movl $3, (%esp) +; X86: calll _f +; X86-NEXT: movl $[[restorebb2]], %eax +; X86-NEXT: addl $8, %esp +; X86-NEXT: popl %ebp +; X86-NEXT: retl + +; X86: L__ehtable$try_catch_catch: +; X86: $handlerMap$0$try_catch_catch: +; X86-NEXT: .long 0 +; X86-NEXT: .long "??_R0H@8" +; X86-NEXT: .long -20 +; X86-NEXT: .long "?catch$[[catch1bb]]@?0?try_catch_catch@4HA" +; X86-NEXT: .long 64 +; X86-NEXT: .long 0 +; X86-NEXT: .long 0 +; X86-NEXT: .long "?catch$[[catch2bb]]@?0?try_catch_catch@4HA" + +; X64-LABEL: try_catch_catch: +; X64: Lfunc_begin0: +; X64: pushq %rbp +; X64: .seh_pushreg 5 +; X64: subq $48, %rsp +; X64: .seh_stackalloc 48 +; X64: leaq 48(%rsp), %rbp +; X64: .seh_setframe 5, 48 +; X64: .seh_endprologue +; X64: movq $-2, -8(%rbp) +; X64: .Ltmp0 +; X64-DAG: leaq -[[local_offs:[0-9]+]](%rbp), %rdx +; X64-DAG: movl $1, %ecx +; X64: callq f +; X64: [[contbb:\.LBB0_[0-9]+]]: # Block address taken +; X64-NEXT: # %try.cont +; X64: addq $48, %rsp +; X64: popq %rbp +; X64: retq + +; X64: "?catch$[[catch1bb:[0-9]+]]@?0?try_catch_catch@4HA": +; X64: LBB0_[[catch1bb]]: # %handler1{{$}} +; X64: movq %rdx, 16(%rsp) +; X64: pushq %rbp +; X64: .seh_pushreg 5 +; X64: subq $32, %rsp +; X64: .seh_stackalloc 32 +; X64: leaq 48(%rdx), %rbp +; X64: .seh_endprologue +; X64-DAG: leaq -[[local_offs]](%rbp), %rdx +; X64-DAG: movl -12(%rbp), %ecx +; X64: callq f +; X64: leaq [[contbb]](%rip), %rax +; X64-NEXT: addq $32, %rsp +; X64-NEXT: popq %rbp +; X64-NEXT: retq + +; X64: "?catch$[[catch2bb:[0-9]+]]@?0?try_catch_catch@4HA": +; X64: LBB0_[[catch2bb]]: # %handler2{{$}} +; X64: movq %rdx, 16(%rsp) +; X64: pushq %rbp +; X64: .seh_pushreg 5 +; X64: subq $32, %rsp +; X64: .seh_stackalloc 32 +; X64: leaq 48(%rdx), %rbp +; X64: .seh_endprologue +; X64-DAG: leaq -[[local_offs]](%rbp), %rdx +; X64-DAG: movl $3, %ecx +; X64: callq f +; X64: leaq [[contbb]](%rip), %rax +; X64-NEXT: addq $32, %rsp +; X64-NEXT: popq %rbp +; X64-NEXT: retq + +; X64: $cppxdata$try_catch_catch: +; X64-NEXT: .long 429065506 +; X64-NEXT: .long 2 +; X64-NEXT: .long ($stateUnwindMap$try_catch_catch)@IMGREL +; X64-NEXT: .long 1 +; X64-NEXT: .long ($tryMap$try_catch_catch)@IMGREL +; X64-NEXT: .long 5 +; X64-NEXT: .long ($ip2state$try_catch_catch)@IMGREL +; X64-NEXT: .long 40 +; X64-NEXT: .long 0 +; X64-NEXT: .long 1 + +; X64: $tryMap$try_catch_catch: +; X64-NEXT: .long 0 +; X64-NEXT: .long 0 +; X64-NEXT: .long 1 +; X64-NEXT: .long 2 +; X64-NEXT: .long ($handlerMap$0$try_catch_catch)@IMGREL + +; X64: $handlerMap$0$try_catch_catch: +; X64-NEXT: .long 0 +; X64-NEXT: .long "??_R0H@8"@IMGREL +; X64-NEXT: .long 36 +; X64-NEXT: .long "?catch$[[catch1bb]]@?0?try_catch_catch@4HA"@IMGREL +; X64-NEXT: .long 56 +; X64-NEXT: .long 64 +; X64-NEXT: .long 0 +; X64-NEXT: .long 0 +; X64-NEXT: .long "?catch$[[catch2bb]]@?0?try_catch_catch@4HA"@IMGREL +; X64-NEXT: .long 56 + +; X64: $ip2state$try_catch_catch: +; X64-NEXT: .long .Lfunc_begin0@IMGREL +; X64-NEXT: .long -1 +; X64-NEXT: .long .Ltmp0@IMGREL+1 +; X64-NEXT: .long 0 +; X64-NEXT: .long .Ltmp1@IMGREL+1 +; X64-NEXT: .long -1 +; X64-NEXT: .long "?catch$[[catch1bb]]@?0?try_catch_catch@4HA"@IMGREL +; X64-NEXT: .long 1 +; X64-NEXT: .long "?catch$[[catch2bb]]@?0?try_catch_catch@4HA"@IMGREL +; X64-NEXT: .long 1 + + +define i32 @branch_to_normal_dest() personality i32 (...)* @__CxxFrameHandler3 { +entry: + invoke void @f(i32 1, i32* null) + to label %try.cont unwind label %catch.dispatch + +catch.dispatch: + %cs1 = catchswitch within none [label %catch] unwind to caller + +catch: + %cp1 = catchpad within %cs1 [i8* null, i32 64, i8* null] + br label %loop + +loop: + %V = call i1 @getbool() [ "funclet"(token %cp1) ] + br i1 %V, label %loop, label %catch.done + +catch.done: + catchret from %cp1 to label %try.cont + +try.cont: + ret i32 0 +} + +; X86-LABEL: _branch_to_normal_dest: +; X86: calll _f + +; X86: [[contbb:LBB1_[0-9]+]]: # %try.cont +; X86: retl + +; X86: [[restorebb:LBB1_[0-9]+]]: # Block address taken +; X86-NEXT: # %catch.done +; X86-NEXT: addl $12, %ebp +; X86: jmp [[contbb]] + +; X86: "?catch$[[catchbb:[0-9]+]]@?0?branch_to_normal_dest@4HA": +; X86: LBB1_[[catchbb]]: # %catch{{$}} +; X86: pushl %ebp +; X86: subl $8, %esp +; X86: addl $12, %ebp +; X86: LBB1_[[loopbb:[0-9]+]]: # %loop +; X86: movl $1, -16(%ebp) +; X86: calll _getbool +; X86: testb $1, %al +; X86: jne LBB1_[[loopbb]] +; X86: # %catch.done +; X86-NEXT: movl $[[restorebb]], %eax +; X86-NEXT: addl $8, %esp +; X86-NEXT: popl %ebp +; X86-NEXT: retl + +; X86: L__ehtable$branch_to_normal_dest: +; X86: $handlerMap$0$branch_to_normal_dest: +; X86-NEXT: .long 64 +; X86-NEXT: .long 0 +; X86-NEXT: .long 0 +; X86-NEXT: .long "?catch$[[catchbb]]@?0?branch_to_normal_dest@4HA" + +; X64-LABEL: branch_to_normal_dest: +; X64: # %entry +; X64: pushq %rbp +; X64: .seh_pushreg 5 +; X64: subq $48, %rsp +; X64: .seh_stackalloc 48 +; X64: leaq 48(%rsp), %rbp +; X64: .seh_setframe 5, 48 +; X64: .seh_endprologue +; X64: .Ltmp[[before_call:[0-9]+]]: +; X64: callq f +; X64: .Ltmp[[after_call:[0-9]+]]: +; X64: [[contbb:\.LBB1_[0-9]+]]: # Block address taken +; X64-NEXT: # %try.cont +; X64: addq $48, %rsp +; X64: popq %rbp +; X64: retq + +; X64: "?catch$[[catchbb:[0-9]+]]@?0?branch_to_normal_dest@4HA": +; X64: LBB1_[[catchbb]]: # %catch{{$}} +; X64: movq %rdx, 16(%rsp) +; X64: pushq %rbp +; X64: .seh_pushreg 5 +; X64: subq $32, %rsp +; X64: .seh_stackalloc 32 +; X64: leaq 48(%rdx), %rbp +; X64: .seh_endprologue +; X64: .LBB1_[[normal_dest_bb:[0-9]+]]: # %loop +; X64: callq getbool +; X64: testb $1, %al +; X64: jne .LBB1_[[normal_dest_bb]] +; X64: # %catch.done +; X64: leaq [[contbb]](%rip), %rax +; X64-NEXT: addq $32, %rsp +; X64-NEXT: popq %rbp +; X64-NEXT: retq + +; X64-LABEL: $cppxdata$branch_to_normal_dest: +; X64-NEXT: .long 429065506 +; X64-NEXT: .long 2 +; X64-NEXT: .long ($stateUnwindMap$branch_to_normal_dest)@IMGREL +; X64-NEXT: .long 1 +; X64-NEXT: .long ($tryMap$branch_to_normal_dest)@IMGREL +; X64-NEXT: .long 4 +; X64-NEXT: .long ($ip2state$branch_to_normal_dest)@IMGREL +; X64-NEXT: .long 40 +; X64-NEXT: .long 0 +; X64-NEXT: .long 1 + +; X64-LABEL: $stateUnwindMap$branch_to_normal_dest: +; X64-NEXT: .long -1 +; X64-NEXT: .long 0 +; X64-NEXT: .long -1 +; X64-NEXT: .long 0 + +; X64-LABEL: $tryMap$branch_to_normal_dest: +; X64-NEXT: .long 0 +; X64-NEXT: .long 0 +; X64-NEXT: .long 1 +; X64-NEXT: .long 1 +; X64-NEXT: .long ($handlerMap$0$branch_to_normal_dest)@IMGREL + +; X64-LABEL: $handlerMap$0$branch_to_normal_dest: +; X64-NEXT: .long 64 +; X64-NEXT: .long 0 +; X64-NEXT: .long 0 +; X64-NEXT: .long "?catch$[[catchbb]]@?0?branch_to_normal_dest@4HA"@IMGREL +; X64-NEXT: .long 56 + +; X64-LABEL: $ip2state$branch_to_normal_dest: +; X64-NEXT: .long .Lfunc_begin1@IMGREL +; X64-NEXT: .long -1 +; X64-NEXT: .long .Ltmp[[before_call]]@IMGREL+1 +; X64-NEXT: .long 0 +; X64-NEXT: .long .Ltmp[[after_call]]@IMGREL+1 +; X64-NEXT: .long -1 +; X64-NEXT: .long "?catch$[[catchbb]]@?0?branch_to_normal_dest@4HA"@IMGREL +; X64-NEXT: .long 1 diff --git a/test/CodeGen/X86/win-cleanuppad.ll b/test/CodeGen/X86/win-cleanuppad.ll new file mode 100644 index 000000000000..4b0a543a876a --- /dev/null +++ b/test/CodeGen/X86/win-cleanuppad.ll @@ -0,0 +1,199 @@ +; RUN: llc -verify-machineinstrs -mtriple=i686-pc-windows-msvc < %s | FileCheck --check-prefix=X86 %s +; RUN: llc -verify-machineinstrs -mtriple=x86_64-pc-windows-msvc < %s | FileCheck --check-prefix=X64 %s + +%struct.Dtor = type { i8 } + +define void @simple_cleanup() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) { +entry: + %o = alloca %struct.Dtor, align 1 + invoke void @f(i32 1) + to label %invoke.cont unwind label %ehcleanup + +invoke.cont: ; preds = %entry + call x86_thiscallcc void @"\01??1Dtor@@QAE@XZ"(%struct.Dtor* %o) #2 + ret void + +ehcleanup: ; preds = %entry + %0 = cleanuppad within none [] + call x86_thiscallcc void @"\01??1Dtor@@QAE@XZ"(%struct.Dtor* %o) #2 [ "funclet"(token %0) ] + cleanupret from %0 unwind to caller +} + +; CHECK: simple_cleanup: # @simple_cleanup +; CHECK: pushq %rbp +; CHECK: subq $48, %rsp +; CHECK: leaq 48(%rsp), %rbp +; CHECK: movq $-2, -8(%rbp) +; CHECK: movl $1, %ecx +; CHECK: callq f +; CHECK: callq "??1Dtor@@QAE@XZ" +; CHECK: nop +; CHECK: addq $48, %rsp +; CHECK: popq %rbp +; CHECK: retq + +; CHECK: "?dtor$2@?0?simple_cleanup@4HA": +; CHECK: callq "??1Dtor@@QAE@XZ" +; CHECK: retq + +; CHECK: $cppxdata$simple_cleanup: +; CHECK-NEXT: .long 429065506 +; CHECK-NEXT: .long 1 +; CHECK-NEXT: .long ($stateUnwindMap$simple_cleanup)@IMGREL +; CHECK-NEXT: .long 0 +; CHECK-NEXT: .long 0 +; CHECK-NEXT: .long 3 +; CHECK-NEXT: .long ($ip2state$simple_cleanup)@IMGREL +; UnwindHelp offset should match the -2 store above +; CHECK-NEXT: .long 40 +; CHECK-NEXT: .long 0 +; CHECK-NEXT: .long 1 + +declare void @f(i32) #0 + +declare i32 @__CxxFrameHandler3(...) + +; Function Attrs: nounwind +declare x86_thiscallcc void @"\01??1Dtor@@QAE@XZ"(%struct.Dtor*) #1 + +define void @nested_cleanup() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) { +entry: + %o1 = alloca %struct.Dtor, align 1 + %o2 = alloca %struct.Dtor, align 1 + invoke void @f(i32 1) + to label %invoke.cont unwind label %cleanup.outer + +invoke.cont: ; preds = %entry + invoke void @f(i32 2) + to label %invoke.cont.1 unwind label %cleanup.inner + +invoke.cont.1: ; preds = %invoke.cont + call x86_thiscallcc void @"\01??1Dtor@@QAE@XZ"(%struct.Dtor* %o2) #2 + invoke void @f(i32 3) + to label %invoke.cont.2 unwind label %cleanup.outer + +invoke.cont.2: ; preds = %invoke.cont.1 + call x86_thiscallcc void @"\01??1Dtor@@QAE@XZ"(%struct.Dtor* %o1) #2 + ret void + +cleanup.inner: ; preds = %invoke.cont + %0 = cleanuppad within none [] + call x86_thiscallcc void @"\01??1Dtor@@QAE@XZ"(%struct.Dtor* %o2) #2 [ "funclet"(token %0) ] + cleanupret from %0 unwind label %cleanup.outer + +cleanup.outer: ; preds = %invoke.cont.1, %cleanup.inner, %entry + %1 = cleanuppad within none [] + call x86_thiscallcc void @"\01??1Dtor@@QAE@XZ"(%struct.Dtor* %o1) #2 [ "funclet"(token %1) ] + cleanupret from %1 unwind to caller +} + +; X86-LABEL: _nested_cleanup: +; X86: movl $1, (%esp) +; X86: calll _f +; X86: movl $2, (%esp) +; X86: calll _f +; X86: movl $3, (%esp) +; X86: calll _f + +; X86: "?dtor$[[cleanup_inner:[0-9]+]]@?0?nested_cleanup@4HA": +; X86: LBB1_[[cleanup_inner]]: # %cleanup.inner{{$}} +; X86: pushl %ebp +; X86: leal {{.*}}(%ebp), %ecx +; X86: calll "??1Dtor@@QAE@XZ" +; X86: popl %ebp +; X86: retl + +; X86: "?dtor$[[cleanup_outer:[0-9]+]]@?0?nested_cleanup@4HA": +; X86: LBB1_[[cleanup_outer]]: # %cleanup.outer{{$}} +; X86: pushl %ebp +; X86: leal {{.*}}(%ebp), %ecx +; X86: calll "??1Dtor@@QAE@XZ" +; X86: popl %ebp +; X86: retl + +; X86: L__ehtable$nested_cleanup: +; X86: .long 429065506 +; X86: .long 2 +; X86: .long ($stateUnwindMap$nested_cleanup) +; X86: .long 0 +; X86: .long 0 +; X86: .long 0 +; X86: .long 0 +; X86: .long 0 +; X86: .long 1 +; X86: $stateUnwindMap$nested_cleanup: +; X86: .long -1 +; X86: .long "?dtor$[[cleanup_outer]]@?0?nested_cleanup@4HA" +; X86: .long 0 +; X86: .long "?dtor$[[cleanup_inner]]@?0?nested_cleanup@4HA" + +; X64-LABEL: nested_cleanup: +; X64: .Lfunc_begin1: +; X64: .Ltmp13: +; X64: movl $1, %ecx +; X64: callq f +; X64: .Ltmp15: +; X64: movl $2, %ecx +; X64: callq f +; X64: .Ltmp16: +; X64: callq "??1Dtor@@QAE@XZ" +; X64: .Ltmp17: +; X64: movl $3, %ecx +; X64: callq f +; X64: .Ltmp18: + +; X64: "?dtor$[[cleanup_inner:[0-9]+]]@?0?nested_cleanup@4HA": +; X64: LBB1_[[cleanup_inner]]: # %cleanup.inner{{$}} +; X64: pushq %rbp +; X64: leaq {{.*}}(%rbp), %rcx +; X64: callq "??1Dtor@@QAE@XZ" +; X64: popq %rbp +; X64: retq + +; X64: .seh_handlerdata +; X64: .text +; X64: .seh_endproc + +; X64: "?dtor$[[cleanup_outer:[0-9]+]]@?0?nested_cleanup@4HA": +; X64: LBB1_[[cleanup_outer]]: # %cleanup.outer{{$}} +; X64: pushq %rbp +; X64: leaq {{.*}}(%rbp), %rcx +; X64: callq "??1Dtor@@QAE@XZ" +; X64: popq %rbp +; X64: retq + +; X64: .section .xdata,"dr" +; X64-NEXT: .align 4 +; X64: $cppxdata$nested_cleanup: +; X64-NEXT: .long 429065506 +; X64-NEXT: .long 2 +; X64-NEXT: .long ($stateUnwindMap$nested_cleanup)@IMGREL +; X64-NEXT: .long 0 +; X64-NEXT: .long 0 +; X64-NEXT: .long 5 +; X64-NEXT: .long ($ip2state$nested_cleanup)@IMGREL +; X64-NEXT: .long 56 +; X64-NEXT: .long 0 +; X64-NEXT: .long 1 + +; X64: $stateUnwindMap$nested_cleanup: +; X64-NEXT: .long -1 +; X64-NEXT: .long "?dtor$[[cleanup_outer]]@?0?nested_cleanup@4HA"@IMGREL +; X64-NEXT: .long 0 +; X64-NEXT: .long "?dtor$[[cleanup_inner]]@?0?nested_cleanup@4HA"@IMGREL + +; X64: $ip2state$nested_cleanup: +; X64-NEXT: .long .Lfunc_begin1@IMGREL +; X64-NEXT: .long -1 +; X64-NEXT: .long .Ltmp13@IMGREL +; X64-NEXT: .long 0 +; X64-NEXT: .long .Ltmp15@IMGREL +; X64-NEXT: .long 1 +; X64-NEXT: .long .Ltmp17@IMGREL +; X64-NEXT: .long 0 +; X64-NEXT: .long .Ltmp18@IMGREL+1 +; X64-NEXT: .long -1 + +attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #2 = { nounwind } diff --git a/test/CodeGen/X86/win-funclet-cfi.ll b/test/CodeGen/X86/win-funclet-cfi.ll new file mode 100644 index 000000000000..2151cdc7bb4b --- /dev/null +++ b/test/CodeGen/X86/win-funclet-cfi.ll @@ -0,0 +1,95 @@ +; RUN: llc -mtriple=x86_64-windows-msvc < %s | FileCheck %s + +target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc" + +define void @"\01?f@@YAXXZ"(i1 %B) personality i32 (...)* @__CxxFrameHandler3 { +entry: + invoke void @g() + to label %unreachable unwind label %cleanupblock + +cleanupblock: + %cleanp = cleanuppad within none [] + call void @g() [ "funclet"(token %cleanp) ] + cleanupret from %cleanp unwind label %catch.dispatch + +catch.dispatch: + %cs1 = catchswitch within none [label %catch] unwind to caller + +catch: + %cp = catchpad within %cs1 [i8* null, i32 64, i8* null] + call void @g() [ "funclet"(token %cp) ] + catchret from %cp to label %try.cont + +try.cont: + ret void + +unreachable: + unreachable +} + + +declare void @g() + +declare i32 @__CxxFrameHandler3(...) + +; Destructors need CFI but they shouldn't use the .seh_handler directive. +; CHECK: "?dtor$[[cleanup:[0-9]+]]@?0??f@@YAXXZ@4HA": +; CHECK: .seh_proc "?dtor$[[cleanup]]@?0??f@@YAXXZ@4HA" +; CHECK-NOT: .seh_handler __CxxFrameHandler3 +; CHECK: LBB0_[[cleanup]]: # %cleanupblock{{$}} + +; Emit CFI for pushing RBP. +; CHECK: movq %rdx, 16(%rsp) +; CHECK: pushq %rbp +; CHECK: .seh_pushreg 5 + +; Emit CFI for allocating from the stack pointer. +; CHECK: subq $32, %rsp +; CHECK: .seh_stackalloc 32 + +; CHECK: leaq 48(%rdx), %rbp +; CHECK-NOT: .seh_setframe + +; Prologue is done, emit the .seh_endprologue directive. +; CHECK: .seh_endprologue + +; Make sure there is a nop after a call if the call precedes the epilogue. +; CHECK: callq g +; CHECK-NEXT: nop + +; Don't emit a reference to the LSDA. +; CHECK: .seh_handlerdata +; CHECK-NOT: .long ("$cppxdata$?f@@YAXXZ")@IMGREL +; CHECK-NEXT: .text +; CHECK: .seh_endproc + +; CHECK: "?catch$[[catch:[0-9]+]]@?0??f@@YAXXZ@4HA": +; CHECK: .seh_proc "?catch$[[catch]]@?0??f@@YAXXZ@4HA" +; CHECK-NEXT: .seh_handler __CxxFrameHandler3, @unwind, @except +; CHECK: LBB0_[[catch]]: # %catch{{$}} + +; Emit CFI for pushing RBP. +; CHECK: movq %rdx, 16(%rsp) +; CHECK: pushq %rbp +; CHECK: .seh_pushreg 5 + +; Emit CFI for allocating from the stack pointer. +; CHECK: subq $32, %rsp +; CHECK: .seh_stackalloc 32 + +; CHECK: leaq 48(%rdx), %rbp +; CHECK-NOT: .seh_setframe + +; Prologue is done, emit the .seh_endprologue directive. +; CHECK: .seh_endprologue + +; Make sure there is at least one instruction after a call before the epilogue. +; CHECK: callq g +; CHECK-NEXT: leaq .LBB0_{{[0-9]+}}(%rip), %rax + +; Emit a reference to the LSDA. +; CHECK: .seh_handlerdata +; CHECK-NEXT: .long ("$cppxdata$?f@@YAXXZ")@IMGREL +; CHECK-NEXT: .text +; CHECK: .seh_endproc diff --git a/test/CodeGen/X86/win-mixed-ehpersonality.ll b/test/CodeGen/X86/win-mixed-ehpersonality.ll new file mode 100644 index 000000000000..f7b6d0702ebe --- /dev/null +++ b/test/CodeGen/X86/win-mixed-ehpersonality.ll @@ -0,0 +1,81 @@ +; RUN: llc -mtriple x86_64-pc-windows-msvc < %s | FileCheck %s + +declare void @maybe_throw() + +@_ZTIi = external constant i8* +@g = external global i32 + +declare i32 @__C_specific_handler(...) +declare i32 @__gxx_personality_seh0(...) +declare i32 @llvm.eh.typeid.for(i8*) readnone nounwind + +define i32 @use_seh() personality i32 (...)* @__C_specific_handler { +entry: + invoke void @maybe_throw() + to label %cont unwind label %lpad + +cont: + ret i32 0 + +lpad: + %cs = catchswitch within none [label %catch] unwind to caller +catch: + %p = catchpad within %cs [i8* bitcast (i32 (i8*, i8*)* @filt_g to i8*)] + catchret from %p to label %ret1 + +ret1: + ret i32 1 +} + +define internal i32 @filt_g(i8*, i8*) { + %g = load i32, i32* @g + ret i32 %g +} + +; CHECK-LABEL: use_seh: +; CHECK: callq maybe_throw +; CHECK: xorl %eax, %eax +; CHECK: .LBB0_[[epilogue:[0-9]+]] +; CHECK: retq +; CHECK: # %catch{{$}} +; CHECK: movl $1, %eax +; CHECK: jmp .LBB0_[[epilogue]] + +; A MinGW64-ish EH style. It could happen if a binary uses both MSVC CRT and +; mingw CRT and is linked with LTO. +define i32 @use_gcc() personality i32 (...)* @__gxx_personality_seh0 { +entry: + invoke void @maybe_throw() + to label %cont unwind label %lpad + +cont: + ret i32 0 + +lpad: + %ehvals = landingpad { i8*, i32 } + cleanup + catch i8* bitcast (i8** @_ZTIi to i8*) + %ehsel = extractvalue { i8*, i32 } %ehvals, 1 + %filt_g_sel = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 (i8*, i8*)* @filt_g to i8*)) + %matches = icmp eq i32 %ehsel, %filt_g_sel + br i1 %matches, label %ret1, label %eh.resume + +ret1: + ret i32 1 + +eh.resume: + resume { i8*, i32 } %ehvals +} + +; CHECK-LABEL: use_gcc: +; CHECK: callq maybe_throw +; CHECK: xorl %eax, %eax +; +; CHECK: # %lpad +; CHECK: cmpl $2, %edx +; CHECK: jne +; +; CHECK: # %ret1 +; CHECK: movl $1, %eax +; +; CHECK: callq _Unwind_Resume diff --git a/test/CodeGen/X86/win32-eh-states.ll b/test/CodeGen/X86/win32-eh-states.ll index 0aae8c4d0189..2777d6644e6a 100644 --- a/test/CodeGen/X86/win32-eh-states.ll +++ b/test/CodeGen/X86/win32-eh-states.ll @@ -1,4 +1,5 @@ -; RUN: llc -mtriple=i686-pc-windows-msvc < %s | FileCheck %s +; RUN: llc -mtriple=i686-pc-windows-msvc < %s | FileCheck %s --check-prefix=X86 +; RUN: llc -mtriple=x86_64-pc-windows-msvc < %s | FileCheck %s --check-prefix=X64 ; Based on this source: ; extern "C" void may_throw(int); @@ -33,82 +34,174 @@ $"\01??_R0H@8" = comdat any define void @f() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) { entry: invoke void @may_throw(i32 1) - to label %invoke.cont unwind label %lpad + to label %invoke.cont unwind label %lpad.1 invoke.cont: ; preds = %entry invoke void @may_throw(i32 2) - to label %try.cont.9 unwind label %lpad.1 + to label %try.cont.9 unwind label %lpad try.cont.9: ; preds = %invoke.cont.3, %invoke.cont, %catch.7 - ; FIXME: Something about our CFG breaks TailDuplication. This empy asm blocks - ; it so we can focus on testing the state numbering. - call void asm sideeffect "", "~{dirflag},~{fpsr},~{flags}"() ret void lpad: ; preds = %catch, %entry - %0 = landingpad { i8*, i32 } - catch %eh.CatchHandlerType* @llvm.eh.handlertype.H.0 - %1 = extractvalue { i8*, i32 } %0, 0 - %2 = extractvalue { i8*, i32 } %0, 1 - br label %catch.dispatch.4 - -lpad.1: ; preds = %invoke.cont - %3 = landingpad { i8*, i32 } - catch i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*) - %4 = extractvalue { i8*, i32 } %3, 0 - %5 = extractvalue { i8*, i32 } %3, 1 - %6 = tail call i32 @llvm.eh.typeid.for(i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*)) #3 - %matches = icmp eq i32 %5, %6 - br i1 %matches, label %catch, label %catch.dispatch.4 - -catch.dispatch.4: ; preds = %lpad.1, %lpad - %exn.slot.0 = phi i8* [ %4, %lpad.1 ], [ %1, %lpad ] - %ehselector.slot.0 = phi i32 [ %5, %lpad.1 ], [ %2, %lpad ] - %.pre = tail call i32 @llvm.eh.typeid.for(i8* bitcast (%eh.CatchHandlerType* @llvm.eh.handlertype.H.0 to i8*)) #3 - %matches6 = icmp eq i32 %ehselector.slot.0, %.pre - br i1 %matches6, label %catch.7, label %eh.resume - -catch.7: ; preds = %catch.dispatch.4 - tail call void @llvm.eh.begincatch(i8* %exn.slot.0, i8* null) #3 - tail call void @may_throw(i32 4) - tail call void @llvm.eh.endcatch() #3 - br label %try.cont.9 + %cs1 = catchswitch within none [label %catch] unwind label %lpad.1 catch: ; preds = %lpad.1 - tail call void @llvm.eh.begincatch(i8* %4, i8* null) #3 - invoke void @may_throw(i32 3) - to label %invoke.cont.3 unwind label %lpad + %p1 = catchpad within %cs1 [%rtti.TypeDescriptor2* @"\01??_R0H@8", i32 0, i8* null] + invoke void @may_throw(i32 3) [ "funclet"(token %p1) ] + to label %invoke.cont.3 unwind label %lpad.1 invoke.cont.3: ; preds = %catch - tail call void @llvm.eh.endcatch() #3 - br label %try.cont.9 + catchret from %p1 to label %try.cont.9 + +lpad.1: ; preds = %invoke.cont + %cs2 = catchswitch within none [label %catch.7] unwind to caller -eh.resume: ; preds = %catch.dispatch.4 - %lpad.val = insertvalue { i8*, i32 } undef, i8* %exn.slot.0, 0 - %lpad.val.12 = insertvalue { i8*, i32 } %lpad.val, i32 %ehselector.slot.0, 1 - resume { i8*, i32 } %lpad.val.12 +catch.7: + %p2 = catchpad within %cs2 [%rtti.TypeDescriptor2* @"\01??_R0H@8", i32 0, i8* null] + call void @may_throw(i32 4) [ "funclet"(token %p2) ] + catchret from %p2 to label %try.cont.9 } -; CHECK-LABEL: _f: -; CHECK: movl $-1, [[state:[-0-9]+]](%ebp) -; CHECK: movl $___ehhandler$f, {{.*}} +; X86-LABEL: _f: +; X86: movl $-1, [[state:[-0-9]+]](%ebp) +; X86: movl $___ehhandler$f, {{.*}} ; -; CHECK: movl $0, [[state]](%ebp) -; CHECK: movl $1, (%esp) -; CHECK: calll _may_throw +; X86: movl $0, [[state]](%ebp) +; X86: movl $1, (%esp) +; X86: calll _may_throw ; -; CHECK: movl $1, [[state]](%ebp) -; CHECK: movl $2, (%esp) -; CHECK: calll _may_throw +; X86: movl $1, [[state]](%ebp) +; X86: movl $2, (%esp) +; X86: calll _may_throw +; +; X86: movl $2, [[state]](%ebp) +; X86: movl $3, (%esp) +; X86: calll _may_throw +; +; X86: movl $3, [[state]](%ebp) +; X86: movl $4, (%esp) +; X86: calll _may_throw + + +; X64-LABEL: f: +; X64-LABEL: $ip2state$f: +; X64-NEXT: .long .Lfunc_begin0@IMGREL +; X64-NEXT: .long -1 +; X64-NEXT: .long .Ltmp{{.*}}@IMGREL+1 +; X64-NEXT: .long 0 +; X64-NEXT: .long .Ltmp{{.*}}@IMGREL+1 +; X64-NEXT: .long 1 +; X64-NEXT: .long .Ltmp{{.*}}@IMGREL+1 +; X64-NEXT: .long -1 +; X64-NEXT: .long "?catch${{.*}}@?0?f@4HA"@IMGREL +; X64-NEXT: .long 2 +; X64-NEXT: .long "?catch${{.*}}@?0?f@4HA"@IMGREL +; X64-NEXT: .long 3 + +; Based on this source: +; extern "C" void may_throw(int); +; struct S { ~S(); }; +; void g() { +; S x; +; try { +; may_throw(-1); +; } catch (...) { +; may_throw(0); +; { +; S y; +; may_throw(1); +; } +; may_throw(2); +; } +; } + +%struct.S = type { i8 } +declare void @"\01??1S@@QEAA@XZ"(%struct.S*) + +define void @g() personality i32 (...)* @__CxxFrameHandler3 { +entry: + %x = alloca %struct.S, align 1 + %y = alloca %struct.S, align 1 + invoke void @may_throw(i32 -1) + to label %unreachable unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %0 = catchswitch within none [label %catch] unwind label %ehcleanup5 + +catch: ; preds = %catch.dispatch + %1 = catchpad within %0 [i8* null, i32 64, i8* null] + invoke void @may_throw(i32 0) [ "funclet"(token %1) ] + to label %invoke.cont unwind label %ehcleanup5 + +invoke.cont: ; preds = %catch + invoke void @may_throw(i32 1) [ "funclet"(token %1) ] + to label %invoke.cont2 unwind label %ehcleanup -; CHECK-LABEL: _f.catch: -; CHECK: movl $4, Lf$frame_escape_{{[0-9]+.*}} -; CHECK: movl $4, (%esp) -; CHECK: calll _may_throw +invoke.cont2: ; preds = %invoke.cont + invoke void @"\01??1S@@QEAA@XZ"(%struct.S* nonnull %y) [ "funclet"(token %1) ] + to label %invoke.cont3 unwind label %ehcleanup5 + +invoke.cont3: ; preds = %invoke.cont2 + invoke void @may_throw(i32 2) [ "funclet"(token %1) ] + to label %invoke.cont4 unwind label %ehcleanup5 + +invoke.cont4: ; preds = %invoke.cont3 + catchret from %1 to label %try.cont + +try.cont: ; preds = %invoke.cont4 + call void @"\01??1S@@QEAA@XZ"(%struct.S* nonnull %x) + ret void -; CHECK-LABEL: _f.catch.1: -; CHECK: movl $3, Lf$frame_escape_{{[0-9]+.*}} -; CHECK: movl $3, (%esp) -; CHECK: calll _may_throw +ehcleanup: ; preds = %invoke.cont + %2 = cleanuppad within %1 [] + call void @"\01??1S@@QEAA@XZ"(%struct.S* nonnull %y) [ "funclet"(token %2) ] + cleanupret from %2 unwind label %ehcleanup5 -; CHECK: .safeseh ___ehhandler$f +ehcleanup5: ; preds = %invoke.cont2, %invoke.cont3, %ehcleanup, %catch, %catch.dispatch + %3 = cleanuppad within none [] + call void @"\01??1S@@QEAA@XZ"(%struct.S* nonnull %x) [ "funclet"(token %3) ] + cleanupret from %3 unwind to caller + +unreachable: ; preds = %entry + unreachable +} + +; X86-LABEL: _g: +; X86: movl $-1, [[state:[-0-9]+]](%ebp) +; X86: movl $___ehhandler$g, {{.*}} +; +; X86: movl $1, [[state]](%ebp) +; X86: movl $-1, (%esp) +; X86: calll _may_throw +; +; X86: movl $2, [[state]](%ebp) +; X86: movl $0, (%esp) +; X86: calll _may_throw +; +; X86: movl $3, [[state]](%ebp) +; X86: movl $1, (%esp) +; X86: calll _may_throw +; +; X86: movl $2, [[state]](%ebp) +; X86: movl $2, (%esp) +; X86: calll _may_throw + +; X64-LABEL: g: +; X64-LABEL: $ip2state$g: +; X64-NEXT: .long .Lfunc_begin1@IMGREL +; X64-NEXT: .long -1 +; X64-NEXT: .long .Ltmp{{.*}}@IMGREL+1 +; X64-NEXT: .long 1 +; X64-NEXT: .long .Ltmp{{.*}}@IMGREL+1 +; X64-NEXT: .long -1 +; X64-NEXT: .long "?catch${{.*}}@?0?g@4HA"@IMGREL +; X64-NEXT: .long 2 +; X64-NEXT: .long .Ltmp{{.*}}@IMGREL+1 +; X64-NEXT: .long 3 +; X64-NEXT: .long .Ltmp{{.*}}@IMGREL+1 +; X64-NEXT: .long 2 + + +; X86: .safeseh ___ehhandler$f +; X86: .safeseh ___ehhandler$g diff --git a/test/CodeGen/X86/win32-eh.ll b/test/CodeGen/X86/win32-eh.ll index 3ee4723ce5f3..73c7b486a55a 100644 --- a/test/CodeGen/X86/win32-eh.ll +++ b/test/CodeGen/X86/win32-eh.ll @@ -15,18 +15,14 @@ define internal i32 @catchall_filt() { define void @use_except_handler3() personality i32 (...)* @_except_handler3 { entry: invoke void @may_throw_or_crash() - to label %cont unwind label %catchall + to label %cont unwind label %lpad cont: ret void -catchall: - %0 = landingpad { i8*, i32 } - catch i8* bitcast (i32 ()* @catchall_filt to i8*) - %1 = extractvalue { i8*, i32 } %0, 1 - %2 = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @catchall_filt to i8*)) #4 - %matches = icmp eq i32 %1, %2 - br i1 %matches, label %cont, label %eh.resume -eh.resume: - resume { i8*, i32 } %0 +lpad: + %cs = catchswitch within none [label %catch] unwind to caller +catch: + %p = catchpad within %cs [i8* bitcast (i32 ()* @catchall_filt to i8*)] + catchret from %p to label %cont } ; CHECK-LABEL: _use_except_handler3: @@ -47,28 +43,25 @@ eh.resume: ; CHECK: movl -28(%ebp), %[[next:[^ ,]*]] ; CHECK: movl %[[next]], %fs:0 ; CHECK: retl +; CHECK: LBB1_2: # %catch{{$}} ; CHECK: .section .xdata,"dr" ; CHECK-LABEL: L__ehtable$use_except_handler3: ; CHECK-NEXT: .long -1 ; CHECK-NEXT: .long _catchall_filt -; CHECK-NEXT: .long Ltmp{{[0-9]+}} +; CHECK-NEXT: .long LBB1_2 define void @use_except_handler4() personality i32 (...)* @_except_handler4 { entry: invoke void @may_throw_or_crash() - to label %cont unwind label %catchall + to label %cont unwind label %lpad cont: ret void -catchall: - %0 = landingpad { i8*, i32 } - catch i8* bitcast (i32 ()* @catchall_filt to i8*) - %1 = extractvalue { i8*, i32 } %0, 1 - %2 = call i32 @llvm.eh.typeid.for(i8* bitcast (i32 ()* @catchall_filt to i8*)) #4 - %matches = icmp eq i32 %1, %2 - br i1 %matches, label %cont, label %eh.resume -eh.resume: - resume { i8*, i32 } %0 +lpad: + %cs = catchswitch within none [label %catch] unwind to caller +catch: + %p = catchpad within %cs [i8* bitcast (i32 ()* @catchall_filt to i8*)] + catchret from %p to label %cont } ; CHECK-LABEL: _use_except_handler4: @@ -89,6 +82,7 @@ eh.resume: ; CHECK: movl -28(%ebp), %[[next:[^ ,]*]] ; CHECK: movl %[[next]], %fs:0 ; CHECK: retl +; CHECK: LBB2_2: # %catch{{$}} ; CHECK: .section .xdata,"dr" ; CHECK-LABEL: L__ehtable$use_except_handler4: @@ -98,20 +92,19 @@ eh.resume: ; CHECK-NEXT: .long 0 ; CHECK-NEXT: .long -2 ; CHECK-NEXT: .long _catchall_filt -; CHECK-NEXT: .long Ltmp{{[0-9]+}} +; CHECK-NEXT: .long LBB2_2 define void @use_CxxFrameHandler3() personality i32 (...)* @__CxxFrameHandler3 { invoke void @may_throw_or_crash() to label %cont unwind label %catchall cont: ret void + catchall: - %ehvals = landingpad { i8*, i32 } - catch i8* null - %ehptr = extractvalue { i8*, i32 } %ehvals, 0 - call void @llvm.eh.begincatch(i8* %ehptr, i8* null) - call void @llvm.eh.endcatch() - br label %cont + %cs = catchswitch within none [label %catch] unwind to caller +catch: + %p = catchpad within %cs [i8* null, i32 64, i8* null] + catchret from %p to label %cont } ; CHECK-LABEL: _use_CxxFrameHandler3: diff --git a/test/CodeGen/X86/win32-pic-jumptable.ll b/test/CodeGen/X86/win32-pic-jumptable.ll index cabd36ae395d..3a8ef2d0b916 100644 --- a/test/CodeGen/X86/win32-pic-jumptable.ll +++ b/test/CodeGen/X86/win32-pic-jumptable.ll @@ -1,16 +1,20 @@ ; RUN: llc < %s -relocation-model=pic | FileCheck %s ; CHECK: calll L0$pb +; CHECK-NEXT: Ltmp{{[0-9]+}}: +; CHECK-NEXT: .cfi_adjust_cfa_offset 4 ; CHECK-NEXT: L0$pb: ; CHECK-NEXT: popl %eax +; CHECK-NEXT: Ltmp{{[0-9]+}}: +; CHECK-NEXT: .cfi_adjust_cfa_offset -4 ; CHECK-NEXT: addl LJTI0_0(,%ecx,4), %eax ; CHECK-NEXT: jmpl *%eax ; CHECK: LJTI0_0: +; CHECK-NEXT: .long LBB0_2-L0$pb +; CHECK-NEXT: .long LBB0_3-L0$pb ; CHECK-NEXT: .long LBB0_4-L0$pb ; CHECK-NEXT: .long LBB0_5-L0$pb -; CHECK-NEXT: .long LBB0_6-L0$pb -; CHECK-NEXT: .long LBB0_7-L0$pb target triple = "i686--windows-itanium" diff --git a/test/CodeGen/X86/win32-seh-catchpad-realign.ll b/test/CodeGen/X86/win32-seh-catchpad-realign.ll new file mode 100644 index 000000000000..23aeea37c117 --- /dev/null +++ b/test/CodeGen/X86/win32-seh-catchpad-realign.ll @@ -0,0 +1,77 @@ +; RUN: llc < %s | FileCheck %s + +; The aligned alloca means that we have to realign the stack, which forces the +; use of ESI to address local variables. + +target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32" +target triple = "i686--windows-msvc" + +; Function Attrs: nounwind +define void @realigned_try() personality i8* bitcast (i32 (...)* @_except_handler3 to i8*) { +entry: + %x = alloca [4 x i32], align 16 + %arrayidx = getelementptr inbounds [4 x i32], [4 x i32]* %x, i32 0, i32 0 + invoke void @useit(i32* %arrayidx) + to label %__try.cont unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %cs1 = catchswitch within none [label %__except.ret] unwind to caller + +__except.ret: ; preds = %catch.dispatch + %pad = catchpad within %cs1 [i8* bitcast (i32 ()* @"\01?filt$0@0@realigned_try@@" to i8*)] + catchret from %pad to label %__try.cont + +__try.cont: ; preds = %entry, %__except.ret + ret void +} + +; Function Attrs: nounwind argmemonly + +; Function Attrs: nounwind +define internal i32 @"\01?filt$0@0@realigned_try@@"() { +entry: + ret i32 1 +} + +declare void @useit(i32*) + +declare i32 @_except_handler3(...) + +; CHECK-LABEL: _realigned_try: +; Prologue +; CHECK: pushl %ebp +; CHECK: movl %esp, %ebp +; CHECK: pushl %ebx +; CHECK: pushl %edi +; CHECK: pushl %esi +; CHECK: andl $-16, %esp +; CHECK: subl $64, %esp +; CHECK: movl %esp, %esi +; Spill EBP +; CHECK: movl %ebp, 12(%esi) +; Spill ESP +; CHECK: movl %esp, 36(%esi) +; The state is stored at ESI+56, the end of the node is ESI+60. +; CHECK: movl $-1, 56(%esi) +; +; __try +; CHECK: calll _useit +; +; Epilogue +; CHECK: LBB0_2: # %__try.cont +; CHECK: leal -12(%ebp), %esp +; CHECK: popl %esi +; CHECK: popl %edi +; CHECK: popl %ebx +; CHECK: popl %ebp +; CHECK: retl +; +; CHECK: LBB0_1: # %__except.ret +; Restore ESP +; CHECK: movl -24(%ebp), %esp +; Recompute ESI by subtracting 60 from the end of the registration node. +; CHECK: leal -60(%ebp), %esi +; Restore EBP +; CHECK: movl 12(%esi), %ebp +; Rejoin normal control flow +; CHECK: jmp LBB0_2 diff --git a/test/CodeGen/X86/win32-seh-catchpad.ll b/test/CodeGen/X86/win32-seh-catchpad.ll new file mode 100644 index 000000000000..224e96f8b8f0 --- /dev/null +++ b/test/CodeGen/X86/win32-seh-catchpad.ll @@ -0,0 +1,231 @@ +; RUN: llc < %s | FileCheck %s + +target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32" +target triple = "i686-pc-windows-msvc" + +define void @try_except() #0 personality i8* bitcast (i32 (...)* @_except_handler3 to i8*) { +entry: + %__exception_code = alloca i32, align 4 + call void (...) @llvm.localescape(i32* %__exception_code) + invoke void @f(i32 1) #3 + to label %invoke.cont unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %cs1 = catchswitch within none [label %__except.ret] unwind to caller + +__except.ret: ; preds = %catch.dispatch + %0 = catchpad within %cs1 [i8* bitcast (i32 ()* @try_except_filter_catchall to i8*)] + catchret from %0 to label %__except + +__except: ; preds = %__except.ret + call void @f(i32 2) + br label %__try.cont + +__try.cont: ; preds = %__except, %invoke.cont + call void @f(i32 3) + ret void + +invoke.cont: ; preds = %entry + br label %__try.cont +} + +; CHECK-LABEL: _try_except: +; Store state #0 +; CHECK: movl $0, -[[state:[0-9]+]](%ebp) +; CHECK: movl $1, (%esp) +; CHECK: calll _f +; CHECK: movl $-1, -[[state]](%ebp) +; CHECK: movl $3, (%esp) +; CHECK: calll _f +; CHECK: retl + +; __except +; CHECK: movl $-1, -[[state]](%ebp) +; CHECK: movl $2, (%esp) +; CHECK: calll _f + +; CHECK: .section .xdata,"dr" +; CHECK: L__ehtable$try_except: +; CHECK: .long -1 # ToState +; CHECK: .long _try_except_filter_catchall # Filter +; CHECK: .long LBB0_1 + +define internal i32 @try_except_filter_catchall() #0 { +entry: + %0 = call i8* @llvm.frameaddress(i32 1) + %1 = call i8* @llvm.x86.seh.recoverfp(i8* bitcast (void ()* @try_except to i8*), i8* %0) + %2 = call i8* @llvm.localrecover(i8* bitcast (void ()* @try_except to i8*), i8* %1, i32 0) + %__exception_code = bitcast i8* %2 to i32* + %3 = getelementptr inbounds i8, i8* %0, i32 -20 + %4 = bitcast i8* %3 to i8** + %5 = load i8*, i8** %4, align 4 + %6 = bitcast i8* %5 to { i32*, i8* }* + %7 = getelementptr inbounds { i32*, i8* }, { i32*, i8* }* %6, i32 0, i32 0 + %8 = load i32*, i32** %7, align 4 + %9 = load i32, i32* %8, align 4 + store i32 %9, i32* %__exception_code, align 4 + ret i32 1 +} + +define void @nested_exceptions() #0 personality i8* bitcast (i32 (...)* @_except_handler3 to i8*) { +entry: + %__exception_code = alloca i32, align 4 + call void (...) @llvm.localescape(i32* %__exception_code) + invoke void @crash() #3 + to label %__try.cont unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %cs1 = catchswitch within none [label %__except.ret] unwind label %catch.dispatch.11 + +__except.ret: ; preds = %catch.dispatch + %0 = catchpad within %cs1 [i8* bitcast (i32 ()* @nested_exceptions_filter_catchall to i8*)] + catchret from %0 to label %__try.cont + +__try.cont: ; preds = %entry, %__except.ret + invoke void @crash() #3 + to label %__try.cont.9 unwind label %catch.dispatch.5 + +catch.dispatch.5: ; preds = %__try.cont + %cs2 = catchswitch within none [label %__except.ret.7] unwind label %catch.dispatch.11 + +__except.ret.7: ; preds = %catch.dispatch.5 + %1 = catchpad within %cs2 [i8* bitcast (i32 ()* @nested_exceptions_filter_catchall to i8*)] + catchret from %1 to label %__try.cont.9 + +__try.cont.9: ; preds = %__try.cont, %__except.ret.7 + invoke void @crash() #3 + to label %__try.cont.15 unwind label %catch.dispatch.11 + +catch.dispatch.11: ; preds = %catchendblock, %catchendblock.6, %__try.cont.9 + %cs3 = catchswitch within none [label %__except.ret.13] unwind label %catch.dispatch.17 + +__except.ret.13: ; preds = %catch.dispatch.11 + %2 = catchpad within %cs3 [i8* bitcast (i32 ()* @nested_exceptions_filter_catchall to i8*)] + catchret from %2 to label %__try.cont.15 + +__try.cont.15: ; preds = %__try.cont.9, %__except.ret.13 + invoke void @crash() #3 + to label %__try.cont.35 unwind label %catch.dispatch.17 + +catch.dispatch.17: ; preds = %catchendblock.12, %__try.cont.15 + %cs4 = catchswitch within none [label %__except.ret.19] unwind to caller + +__except.ret.19: ; preds = %catch.dispatch.17 + %3 = catchpad within %cs4 [i8* bitcast (i32 ()* @nested_exceptions_filter_catchall to i8*)] + catchret from %3 to label %__except.20 + +__except.20: ; preds = %__except.ret.19 + invoke void @crash() #3 + to label %__try.cont.27 unwind label %catch.dispatch.23 + +catch.dispatch.23: ; preds = %__except.20 + %cs5 = catchswitch within none [label %__except.ret.25] unwind to caller + +__except.ret.25: ; preds = %catch.dispatch.23 + %4 = catchpad within %cs5 [i8* bitcast (i32 ()* @nested_exceptions_filter_catchall to i8*)] + catchret from %4 to label %__try.cont.27 + +__try.cont.27: ; preds = %__except.20, %__except.ret.25 + invoke void @crash() #3 + to label %__try.cont.35 unwind label %catch.dispatch.30 + +catch.dispatch.30: ; preds = %__try.cont.27 + %cs6 = catchswitch within none [label %__except.ret.32] unwind to caller + +__except.ret.32: ; preds = %catch.dispatch.30 + %5 = catchpad within %cs6 [i8* bitcast (i32 ()* @nested_exceptions_filter_catchall to i8*)] + catchret from %5 to label %__try.cont.35 + +__try.cont.35: ; preds = %__try.cont.15, %__try.cont.27, %__except.ret.32 + ret void +} + +; This table is equivalent to the one produced by MSVC, even if it isn't in +; quite the same order. + +; CHECK-LABEL: _nested_exceptions: +; CHECK: L__ehtable$nested_exceptions: +; CHECK: .long -1 +; CHECK: .long _nested_exceptions_filter_catchall +; CHECK: .long LBB +; CHECK: .long 0 +; CHECK: .long _nested_exceptions_filter_catchall +; CHECK: .long LBB +; CHECK: .long 1 +; CHECK: .long _nested_exceptions_filter_catchall +; CHECK: .long LBB +; CHECK: .long 1 +; CHECK: .long _nested_exceptions_filter_catchall +; CHECK: .long LBB +; CHECK: .long -1 +; CHECK: .long _nested_exceptions_filter_catchall +; CHECK: .long LBB +; CHECK: .long -1 +; CHECK: .long _nested_exceptions_filter_catchall +; CHECK: .long LBB + +declare void @crash() #0 + +define internal i32 @nested_exceptions_filter_catchall() #0 { +entry: + %0 = call i8* @llvm.frameaddress(i32 1) + %1 = call i8* @llvm.x86.seh.recoverfp(i8* bitcast (void ()* @nested_exceptions to i8*), i8* %0) + %2 = call i8* @llvm.localrecover(i8* bitcast (void ()* @nested_exceptions to i8*), i8* %1, i32 0) + %__exception_code3 = bitcast i8* %2 to i32* + %3 = getelementptr inbounds i8, i8* %0, i32 -20 + %4 = bitcast i8* %3 to i8** + %5 = load i8*, i8** %4, align 4 + %6 = bitcast i8* %5 to { i32*, i8* }* + %7 = getelementptr inbounds { i32*, i8* }, { i32*, i8* }* %6, i32 0, i32 0 + %8 = load i32*, i32** %7, align 4 + %9 = load i32, i32* %8, align 4 + store i32 %9, i32* %__exception_code3, align 4 + ret i32 1 +} + +define void @code_in_catchpad() #0 personality i8* bitcast (i32 (...)* @_except_handler3 to i8*) { +entry: + invoke void @f(i32 1) #3 + to label %__except unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %cs1 = catchswitch within none [label %__except.ret] unwind to caller + +__except.ret: ; preds = %catch.dispatch + %0 = catchpad within %cs1 [i8* bitcast (i32 ()* @try_except_filter_catchall to i8*)] + call void @f(i32 2) [ "funclet"(token %0) ] + catchret from %0 to label %__except + +__except: + ret void +} + +; CHECK-LABEL: _code_in_catchpad: +; CHECK: # %__except.ret +; CHECK-NEXT: movl -24(%ebp), %esp +; CHECK-NEXT: addl $12, %ebp +; CHECK-NEXT: movl $-1, -16(%ebp) +; CHECK-NEXT: movl $2, (%esp) +; CHECK-NEXT: calll _f + + +; Function Attrs: nounwind readnone +declare i8* @llvm.frameaddress(i32) #1 + +; Function Attrs: nounwind readnone +declare i8* @llvm.x86.seh.recoverfp(i8*, i8*) #1 + +; Function Attrs: nounwind readnone +declare i8* @llvm.localrecover(i8*, i8*, i32) #1 + +declare void @f(i32) #0 + +declare i32 @_except_handler3(...) + +; Function Attrs: nounwind +declare void @llvm.localescape(...) #2 + +attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { nounwind readnone } +attributes #2 = { nounwind } +attributes #3 = { noinline } diff --git a/test/CodeGen/X86/win32-seh-nested-finally.ll b/test/CodeGen/X86/win32-seh-nested-finally.ll new file mode 100644 index 000000000000..c283a35d70cf --- /dev/null +++ b/test/CodeGen/X86/win32-seh-nested-finally.ll @@ -0,0 +1,80 @@ +; RUN: llc < %s | FileCheck %s + +target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32" +target triple = "i686-pc-windows-msvc" + +define void @nested_finally() #0 personality i8* bitcast (i32 (...)* @_except_handler3 to i8*) { +entry: + invoke void @f(i32 1) #3 + to label %invoke.cont unwind label %ehcleanup + +invoke.cont: ; preds = %entry + invoke void @f(i32 2) #3 + to label %invoke.cont.1 unwind label %ehcleanup.3 + +invoke.cont.1: ; preds = %invoke.cont + call void @f(i32 3) #3 + ret void + +ehcleanup: ; preds = %entry + %0 = cleanuppad within none [] + invoke void @f(i32 2) #3 [ "funclet"(token %0) ] + to label %invoke.cont.2 unwind label %ehcleanup.3 + +invoke.cont.2: ; preds = %ehcleanup + cleanupret from %0 unwind label %ehcleanup.3 + +ehcleanup.3: ; preds = %invoke.cont.2, %ehcleanup.end, %invoke.cont + %1 = cleanuppad within none [] + call void @f(i32 3) #3 [ "funclet"(token %1) ] + cleanupret from %1 unwind to caller +} + +declare void @f(i32) #0 + +declare i32 @_except_handler3(...) + +attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { noinline nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #2 = { nounwind readnone } +attributes #3 = { noinline } + +; CHECK: _nested_finally: +; CHECK: movl $-1, -[[state:[0-9]+]](%ebp) +; CHECK: movl {{.*}}, %fs:0 +; CHECK: movl $1, -[[state]](%ebp) +; CHECK: movl $1, (%esp) +; CHECK: calll _f +; CHECK: movl $0, -[[state]](%ebp) +; CHECK: movl $2, (%esp) +; CHECK: calll _f +; CHECK: movl $-1, -[[state]](%ebp) +; CHECK: movl $3, (%esp) +; CHECK: calll _f +; CHECK: retl + +; CHECK: LBB0_[[inner:[0-9]+]]: # %ehcleanup +; CHECK: pushl %ebp +; CHECK: addl $12, %ebp +; CHECK: movl $0, -[[state]](%ebp) +; CHECK: movl $2, (%esp) +; CHECK: calll _f +; CHECK: popl %ebp +; CHECK: retl + +; CHECK: LBB0_[[outer:[0-9]+]]: # %ehcleanup.3 +; CHECK: pushl %ebp +; CHECK: addl $12, %ebp +; CHECK: movl $-1, -[[state]](%ebp) +; CHECK: movl $3, (%esp) +; CHECK: calll _f +; CHECK: popl %ebp +; CHECK: retl + +; CHECK: L__ehtable$nested_finally: +; CHECK: .long -1 # ToState +; CHECK: .long 0 # Null +; CHECK: .long "?dtor$[[outer]]@?0?nested_finally@4HA" # FinallyFunclet +; CHECK: .long 0 # ToState +; CHECK: .long 0 # Null +; CHECK: .long "?dtor$[[inner]]@?0?nested_finally@4HA" # FinallyFunclet diff --git a/test/CodeGen/X86/win32-spill-xmm.ll b/test/CodeGen/X86/win32-spill-xmm.ll new file mode 100644 index 000000000000..0db97cfe20f0 --- /dev/null +++ b/test/CodeGen/X86/win32-spill-xmm.ll @@ -0,0 +1,40 @@ +; RUN: llc -mcpu=generic -mtriple=i686-pc-windows-msvc -mattr=+sse < %s | FileCheck %s + +; Check proper alignment of spilled vector + +; CHECK-LABEL: spill_ok +; CHECK: subl $32, %esp +; CHECK: movaps %xmm3, (%esp) +; CHECK: movl $0, 16(%esp) +; CHECK: calll _bar +define void @spill_ok(i32, <16 x float> *) { +entry: + %2 = alloca i32, i32 %0 + %3 = load <16 x float>, <16 x float> * %1, align 64 + tail call void @bar(<16 x float> %3, i32 0) nounwind + ret void +} + +declare void @bar(<16 x float> %a, i32 %b) + +; Check that proper alignment of spilled vector does not affect vargs + +; CHECK-LABEL: vargs_not_affected +; CHECK: leal 28(%ebp), %eax +define i32 @vargs_not_affected(<4 x float> %v, i8* %f, ...) { +entry: + %ap = alloca i8*, align 4 + %0 = bitcast i8** %ap to i8* + call void @llvm.va_start(i8* %0) + %argp.cur = load i8*, i8** %ap, align 4 + %argp.next = getelementptr inbounds i8, i8* %argp.cur, i32 4 + store i8* %argp.next, i8** %ap, align 4 + %1 = bitcast i8* %argp.cur to i32* + %2 = load i32, i32* %1, align 4 + call void @llvm.va_end(i8* %0) + ret i32 %2 +} + +declare void @llvm.va_start(i8*) + +declare void @llvm.va_end(i8*) diff --git a/test/CodeGen/X86/win64_frame.ll b/test/CodeGen/X86/win64_frame.ll index 477b3144d9e7..27d78dbe5479 100644 --- a/test/CodeGen/X86/win64_frame.ll +++ b/test/CodeGen/X86/win64_frame.ll @@ -1,4 +1,5 @@ -; RUN: llc < %s -mtriple=x86_64-pc-win32 | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-pc-win32 | FileCheck %s --check-prefix=CHECK --check-prefix=PUSHF +; RUN: llc < %s -mtriple=x86_64-pc-win32 -mattr=+sahf | FileCheck %s --check-prefix=SAHF define i32 @f1(i32 %p1, i32 %p2, i32 %p3, i32 %p4, i32 %p5) "no-frame-pointer-elim"="true" { ; CHECK-LABEL: f1: @@ -118,6 +119,73 @@ define i32 @f8(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) "no-frame-pointer-elim"=" ; CHECK: leaq 224(%rbp), %rsp } +define i64 @f9() { +entry: + ; CHECK-LABEL: f9: + ; CHECK: pushq %rbp + ; CHECK: .seh_pushreg 5 + ; CHECK-NEXT: movq %rsp, %rbp + ; CHECK: .seh_setframe 5, 0 + ; CHECK: .seh_endprologue + + %call = call i64 asm sideeffect "pushf\0A\09popq $0\0A", "=r,~{dirflag},~{fpsr},~{flags}"() + ; CHECK-NEXT: #APP + ; CHECK-NEXT: pushfq + ; CHECK-NEXT: popq %rax + ; CHECK: #NO_APP + + ret i64 %call + ; CHECK-NEXT: popq %rbp + ; CHECK-NEXT: retq +} + +declare i64 @dummy() + +define i64 @f10(i64* %foo, i64 %bar, i64 %baz) { + ; CHECK-LABEL: f10: + ; CHECK: pushq %rbp + ; CHECK: .seh_pushreg 5 + ; CHECK: pushq %rsi + ; CHECK: .seh_pushreg 6 + ; CHECK: pushq %rdi + ; CHECK: .seh_pushreg 7 + ; CHECK: subq $32, %rsp + ; CHECK: .seh_stackalloc 32 + ; CHECK: leaq 32(%rsp), %rbp + ; CHECK: .seh_setframe 5, 32 + ; CHECK: .seh_endprologue + + %cx = cmpxchg i64* %foo, i64 %bar, i64 %baz seq_cst seq_cst + ; PUSHF: lock cmpxchgq + ; PUSHF-NEXT: pushfq + ; PUSHF-NEXT: popq %[[REG:.*]] + ; SAHF: lock cmpxchgq + ; SAHF-NEXT: seto %al + ; SAHF-NEXT: lahf + + %v = extractvalue { i64, i1 } %cx, 0 + %p = extractvalue { i64, i1 } %cx, 1 + + %call = call i64 @dummy() + ; PUSHF: callq dummy + ; PUSHF-NEXT: pushq %[[REG]] + ; PUSHF-NEXT: popfq + ; SAHF: callq dummy + ; SAHF-NEXT: pushq + ; SAHF: addb $127, %al + ; SAHF-NEXT: sahf + ; SAHF-NEXT: popq + + %sel = select i1 %p, i64 %call, i64 %bar + ; CHECK-NEXT: cmovneq + + ret i64 %sel + ; CHECK-NEXT: addq $32, %rsp + ; CHECK-NEXT: popq %rdi + ; CHECK-NEXT: popq %rsi + ; CHECK-NEXT: popq %rbp +} + declare i8* @llvm.returnaddress(i32) nounwind readnone declare void @llvm.va_start(i8*) nounwind diff --git a/test/CodeGen/X86/win64_sibcall.ll b/test/CodeGen/X86/win64_sibcall.ll new file mode 100644 index 000000000000..4001f638c2ab --- /dev/null +++ b/test/CodeGen/X86/win64_sibcall.ll @@ -0,0 +1,38 @@ +; RUN: llc < %s -mtriple=x86_64-pc-win32-coreclr | FileCheck %s -check-prefix=WIN_X64 +; RUN: llc < %s -mtriple=x86_64-pc-linux | FileCheck %s -check-prefix=LINUX + +%Object = type <{ [0 x i64*]* }> + +define void @C1(%Object addrspace(1)* %param0) gc "coreclr" { +entry: + +; WIN_X64: # BB#0: +; WIN_X64: pushq %rax +; LINUX: # BB#0: # %entry +; LINUX: movq $0, -8(%rsp) + + %this = alloca %Object addrspace(1)* + store %Object addrspace(1)* null, %Object addrspace(1)** %this + store %Object addrspace(1)* %param0, %Object addrspace(1)** %this + br label %0 + +;