aboutsummaryrefslogtreecommitdiff
path: root/lib/Target/X86
diff options
context:
space:
mode:
Diffstat (limited to 'lib/Target/X86')
-rw-r--r--lib/Target/X86/X86FastISel.cpp15
-rw-r--r--lib/Target/X86/X86ISelDAGToDAG.cpp8
-rw-r--r--lib/Target/X86/X86ISelLowering.cpp412
-rw-r--r--lib/Target/X86/X86ISelLowering.h10
-rw-r--r--lib/Target/X86/X86InstrCompiler.td10
-rw-r--r--lib/Target/X86/X86InstrInfo.td10
-rw-r--r--lib/Target/X86/X86InstructionSelector.cpp98
-rw-r--r--lib/Target/X86/X86LegalizerInfo.cpp20
-rw-r--r--lib/Target/X86/X86OptimizeLEAs.cpp54
-rw-r--r--lib/Target/X86/X86SelectionDAGInfo.cpp106
-rw-r--r--lib/Target/X86/X86Subtarget.cpp11
-rw-r--r--lib/Target/X86/X86Subtarget.h9
-rw-r--r--lib/Target/X86/X86TargetMachine.cpp12
13 files changed, 400 insertions, 375 deletions
diff --git a/lib/Target/X86/X86FastISel.cpp b/lib/Target/X86/X86FastISel.cpp
index b8477810b4c9c..fd11b671dd9b7 100644
--- a/lib/Target/X86/X86FastISel.cpp
+++ b/lib/Target/X86/X86FastISel.cpp
@@ -3074,16 +3074,13 @@ bool X86FastISel::fastLowerArguments() {
// Only handle simple cases. i.e. Up to 6 i32/i64 scalar arguments.
unsigned GPRCnt = 0;
unsigned FPRCnt = 0;
- unsigned Idx = 0;
for (auto const &Arg : F->args()) {
- // The first argument is at index 1.
- ++Idx;
- if (F->getAttributes().hasAttribute(Idx, Attribute::ByVal) ||
- F->getAttributes().hasAttribute(Idx, Attribute::InReg) ||
- F->getAttributes().hasAttribute(Idx, Attribute::StructRet) ||
- F->getAttributes().hasAttribute(Idx, Attribute::SwiftSelf) ||
- F->getAttributes().hasAttribute(Idx, Attribute::SwiftError) ||
- F->getAttributes().hasAttribute(Idx, Attribute::Nest))
+ if (Arg.hasAttribute(Attribute::ByVal) ||
+ Arg.hasAttribute(Attribute::InReg) ||
+ Arg.hasAttribute(Attribute::StructRet) ||
+ Arg.hasAttribute(Attribute::SwiftSelf) ||
+ Arg.hasAttribute(Attribute::SwiftError) ||
+ Arg.hasAttribute(Attribute::Nest))
return false;
Type *ArgTy = Arg.getType();
diff --git a/lib/Target/X86/X86ISelDAGToDAG.cpp b/lib/Target/X86/X86ISelDAGToDAG.cpp
index 2d788bf0cf994..12a10bf3072ff 100644
--- a/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -31,6 +31,7 @@
#include "llvm/IR/Type.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
@@ -419,6 +420,7 @@ X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const {
case ISD::ADD:
case ISD::ADDC:
case ISD::ADDE:
+ case ISD::ADDCARRY:
case ISD::AND:
case ISD::OR:
case ISD::XOR: {
@@ -1070,9 +1072,9 @@ static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N,
}
APInt MaskedHighBits =
APInt::getHighBitsSet(X.getSimpleValueType().getSizeInBits(), MaskLZ);
- APInt KnownZero, KnownOne;
- DAG.computeKnownBits(X, KnownZero, KnownOne);
- if (MaskedHighBits != KnownZero) return true;
+ KnownBits Known;
+ DAG.computeKnownBits(X, Known);
+ if (MaskedHighBits != Known.Zero) return true;
// We've identified a pattern that can be transformed into a single shift
// and an addressing mode. Make it so.
diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp
index ada46643a5fef..6092fd2bfd691 100644
--- a/lib/Target/X86/X86ISelLowering.cpp
+++ b/lib/Target/X86/X86ISelLowering.cpp
@@ -52,6 +52,7 @@
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetLowering.h"
#include "llvm/Target/TargetOptions.h"
@@ -784,30 +785,18 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SMIN, MVT::v8i16, Legal);
setOperationAction(ISD::UMIN, MVT::v16i8, Legal);
- setOperationAction(ISD::SETCC, MVT::v2i64, Custom);
- setOperationAction(ISD::SETCC, MVT::v16i8, Custom);
- setOperationAction(ISD::SETCC, MVT::v8i16, Custom);
- setOperationAction(ISD::SETCC, MVT::v4i32, Custom);
-
- setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom);
- setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom);
- setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Custom);
setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
- setOperationAction(ISD::CTPOP, MVT::v16i8, Custom);
- setOperationAction(ISD::CTPOP, MVT::v8i16, Custom);
- setOperationAction(ISD::CTPOP, MVT::v4i32, Custom);
- setOperationAction(ISD::CTPOP, MVT::v2i64, Custom);
-
- setOperationAction(ISD::CTTZ, MVT::v16i8, Custom);
- setOperationAction(ISD::CTTZ, MVT::v8i16, Custom);
- setOperationAction(ISD::CTTZ, MVT::v4i32, Custom);
- setOperationAction(ISD::CTTZ, MVT::v2i64, Custom);
+ for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
+ setOperationAction(ISD::SETCC, VT, Custom);
+ setOperationAction(ISD::CTPOP, VT, Custom);
+ setOperationAction(ISD::CTTZ, VT, Custom);
+ }
- // Custom lower build_vector, vector_shuffle, and extract_vector_elt.
for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
+ setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
setOperationAction(ISD::VSELECT, VT, Custom);
@@ -882,18 +871,12 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i32, Custom);
setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i16, Custom);
- for (auto VT : { MVT::v8i16, MVT::v16i8 }) {
- setOperationAction(ISD::SRL, VT, Custom);
- setOperationAction(ISD::SHL, VT, Custom);
- setOperationAction(ISD::SRA, VT, Custom);
- }
-
- // In the customized shift lowering, the legal cases in AVX2 will be
- // recognized.
- for (auto VT : { MVT::v4i32, MVT::v2i64 }) {
- setOperationAction(ISD::SRL, VT, Custom);
- setOperationAction(ISD::SHL, VT, Custom);
- setOperationAction(ISD::SRA, VT, Custom);
+ // In the customized shift lowering, the legal v4i32/v2i64 cases
+ // in AVX2 will be recognized.
+ for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
+ setOperationAction(ISD::SRL, VT, Custom);
+ setOperationAction(ISD::SHL, VT, Custom);
+ setOperationAction(ISD::SRA, VT, Custom);
}
}
@@ -935,13 +918,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// SSE41 brings specific instructions for doing vector sign extend even in
// cases where we don't have SRA.
- setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v2i64, Legal);
- setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i32, Legal);
- setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i16, Legal);
-
- setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, MVT::v2i64, Legal);
- setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, MVT::v4i32, Legal);
- setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, MVT::v8i16, Legal);
+ for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
+ setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Legal);
+ setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Legal);
+ }
for (MVT VT : MVT::integer_vector_valuetypes()) {
setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Custom);
@@ -950,19 +930,14 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
// SSE41 also has vector sign/zero extending loads, PMOV[SZ]X
- setLoadExtAction(ISD::SEXTLOAD, MVT::v8i16, MVT::v8i8, Legal);
- setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i8, Legal);
- setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i8, Legal);
- setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i16, Legal);
- setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i16, Legal);
- setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i32, Legal);
-
- setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i16, MVT::v8i8, Legal);
- setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i8, Legal);
- setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i8, Legal);
- setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i16, Legal);
- setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i16, Legal);
- setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i32, Legal);
+ for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
+ setLoadExtAction(LoadExtOp, MVT::v8i16, MVT::v8i8, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i8, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i8, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i16, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i16, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i32, Legal);
+ }
// i8 vectors are custom because the source register and source
// source memory operand types are not the same width.
@@ -1026,36 +1001,31 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
for (MVT VT : MVT::fp_vector_valuetypes())
setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4f32, Legal);
- for (auto VT : { MVT::v32i8, MVT::v16i16 }) {
+ // In the customized shift lowering, the legal v8i32/v4i64 cases
+ // in AVX2 will be recognized.
+ for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
setOperationAction(ISD::SRL, VT, Custom);
setOperationAction(ISD::SHL, VT, Custom);
setOperationAction(ISD::SRA, VT, Custom);
}
- setOperationAction(ISD::SETCC, MVT::v32i8, Custom);
- setOperationAction(ISD::SETCC, MVT::v16i16, Custom);
- setOperationAction(ISD::SETCC, MVT::v8i32, Custom);
- setOperationAction(ISD::SETCC, MVT::v4i64, Custom);
-
setOperationAction(ISD::SELECT, MVT::v4f64, Custom);
setOperationAction(ISD::SELECT, MVT::v4i64, Custom);
setOperationAction(ISD::SELECT, MVT::v8f32, Custom);
- setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom);
- setOperationAction(ISD::SIGN_EXTEND, MVT::v8i32, Custom);
- setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom);
- setOperationAction(ISD::ZERO_EXTEND, MVT::v4i64, Custom);
- setOperationAction(ISD::ZERO_EXTEND, MVT::v8i32, Custom);
- setOperationAction(ISD::ZERO_EXTEND, MVT::v16i16, Custom);
- setOperationAction(ISD::ANY_EXTEND, MVT::v4i64, Custom);
- setOperationAction(ISD::ANY_EXTEND, MVT::v8i32, Custom);
- setOperationAction(ISD::ANY_EXTEND, MVT::v16i16, Custom);
+ for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
+ setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
+ setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
+ setOperationAction(ISD::ANY_EXTEND, VT, Custom);
+ }
+
setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom);
setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom);
setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom);
setOperationAction(ISD::BITREVERSE, MVT::v32i8, Custom);
for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
+ setOperationAction(ISD::SETCC, VT, Custom);
setOperationAction(ISD::CTPOP, VT, Custom);
setOperationAction(ISD::CTTZ, VT, Custom);
setOperationAction(ISD::CTLZ, VT, Custom);
@@ -1103,27 +1073,14 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom);
// AVX2 also has wider vector sign/zero extending loads, VPMOV[SZ]X
- setLoadExtAction(ISD::SEXTLOAD, MVT::v16i16, MVT::v16i8, Legal);
- setLoadExtAction(ISD::SEXTLOAD, MVT::v8i32, MVT::v8i8, Legal);
- setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i8, Legal);
- setLoadExtAction(ISD::SEXTLOAD, MVT::v8i32, MVT::v8i16, Legal);
- setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i16, Legal);
- setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i32, Legal);
-
- setLoadExtAction(ISD::ZEXTLOAD, MVT::v16i16, MVT::v16i8, Legal);
- setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i32, MVT::v8i8, Legal);
- setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i8, Legal);
- setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i32, MVT::v8i16, Legal);
- setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i16, Legal);
- setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i32, Legal);
- }
-
- // In the customized shift lowering, the legal cases in AVX2 will be
- // recognized.
- for (auto VT : { MVT::v8i32, MVT::v4i64 }) {
- setOperationAction(ISD::SRL, VT, Custom);
- setOperationAction(ISD::SHL, VT, Custom);
- setOperationAction(ISD::SRA, VT, Custom);
+ for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
+ setLoadExtAction(LoadExtOp, MVT::v16i16, MVT::v16i8, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i8, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i8, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i16, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i16, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i32, Legal);
+ }
}
for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
@@ -1272,19 +1229,12 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::VSELECT, MVT::v8i1, Expand);
setOperationAction(ISD::VSELECT, MVT::v16i1, Expand);
if (Subtarget.hasDQI()) {
- setOperationAction(ISD::SINT_TO_FP, MVT::v8i64, Legal);
- setOperationAction(ISD::SINT_TO_FP, MVT::v4i64, Legal);
- setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal);
- setOperationAction(ISD::UINT_TO_FP, MVT::v8i64, Legal);
- setOperationAction(ISD::UINT_TO_FP, MVT::v4i64, Legal);
- setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal);
- setOperationAction(ISD::FP_TO_SINT, MVT::v8i64, Legal);
- setOperationAction(ISD::FP_TO_SINT, MVT::v4i64, Legal);
- setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal);
- setOperationAction(ISD::FP_TO_UINT, MVT::v8i64, Legal);
- setOperationAction(ISD::FP_TO_UINT, MVT::v4i64, Legal);
- setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal);
-
+ for (auto VT : { MVT::v2i64, MVT::v4i64, MVT::v8i64 }) {
+ setOperationAction(ISD::SINT_TO_FP, VT, Legal);
+ setOperationAction(ISD::UINT_TO_FP, VT, Legal);
+ setOperationAction(ISD::FP_TO_SINT, VT, Legal);
+ setOperationAction(ISD::FP_TO_UINT, VT, Legal);
+ }
if (Subtarget.hasVLX()) {
// Fast v2f32 SINT_TO_FP( v2i32 ) custom conversion.
setOperationAction(ISD::SINT_TO_FP, MVT::v2f32, Custom);
@@ -1334,11 +1284,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom);
for (auto VT : { MVT::v16f32, MVT::v8f64 }) {
- setOperationAction(ISD::FFLOOR, VT, Legal);
- setOperationAction(ISD::FCEIL, VT, Legal);
- setOperationAction(ISD::FTRUNC, VT, Legal);
- setOperationAction(ISD::FRINT, VT, Legal);
- setOperationAction(ISD::FNEARBYINT, VT, Legal);
+ setOperationAction(ISD::FFLOOR, VT, Legal);
+ setOperationAction(ISD::FCEIL, VT, Legal);
+ setOperationAction(ISD::FTRUNC, VT, Legal);
+ setOperationAction(ISD::FRINT, VT, Legal);
+ setOperationAction(ISD::FNEARBYINT, VT, Legal);
}
setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i64, Custom);
@@ -1357,7 +1307,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SETCC, MVT::v16i1, Custom);
setOperationAction(ISD::SETCC, MVT::v8i1, Custom);
- setOperationAction(ISD::MUL, MVT::v8i64, Custom);
+ setOperationAction(ISD::MUL, MVT::v8i64, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i1, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i1, Custom);
@@ -1372,15 +1322,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SELECT, MVT::v16i1, Custom);
setOperationAction(ISD::SELECT, MVT::v8i1, Custom);
- setOperationAction(ISD::SMAX, MVT::v16i32, Legal);
- setOperationAction(ISD::SMAX, MVT::v8i64, Legal);
- setOperationAction(ISD::UMAX, MVT::v16i32, Legal);
- setOperationAction(ISD::UMAX, MVT::v8i64, Legal);
- setOperationAction(ISD::SMIN, MVT::v16i32, Legal);
- setOperationAction(ISD::SMIN, MVT::v8i64, Legal);
- setOperationAction(ISD::UMIN, MVT::v16i32, Legal);
- setOperationAction(ISD::UMIN, MVT::v8i64, Legal);
-
setOperationAction(ISD::ADD, MVT::v8i1, Custom);
setOperationAction(ISD::ADD, MVT::v16i1, Custom);
setOperationAction(ISD::SUB, MVT::v8i1, Custom);
@@ -1391,12 +1332,16 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::MUL, MVT::v16i32, Legal);
for (auto VT : { MVT::v16i32, MVT::v8i64 }) {
- setOperationAction(ISD::ABS, VT, Legal);
- setOperationAction(ISD::SRL, VT, Custom);
- setOperationAction(ISD::SHL, VT, Custom);
- setOperationAction(ISD::SRA, VT, Custom);
- setOperationAction(ISD::CTPOP, VT, Custom);
- setOperationAction(ISD::CTTZ, VT, Custom);
+ setOperationAction(ISD::SMAX, VT, Legal);
+ setOperationAction(ISD::UMAX, VT, Legal);
+ setOperationAction(ISD::SMIN, VT, Legal);
+ setOperationAction(ISD::UMIN, VT, Legal);
+ setOperationAction(ISD::ABS, VT, Legal);
+ setOperationAction(ISD::SRL, VT, Custom);
+ setOperationAction(ISD::SHL, VT, Custom);
+ setOperationAction(ISD::SRA, VT, Custom);
+ setOperationAction(ISD::CTPOP, VT, Custom);
+ setOperationAction(ISD::CTTZ, VT, Custom);
}
// Need to promote to 64-bit even though we have 32-bit masked instructions
@@ -1540,15 +1485,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::VSELECT, MVT::v64i1, Expand);
setOperationAction(ISD::BITREVERSE, MVT::v64i8, Custom);
- setOperationAction(ISD::SMAX, MVT::v64i8, Legal);
- setOperationAction(ISD::SMAX, MVT::v32i16, Legal);
- setOperationAction(ISD::UMAX, MVT::v64i8, Legal);
- setOperationAction(ISD::UMAX, MVT::v32i16, Legal);
- setOperationAction(ISD::SMIN, MVT::v64i8, Legal);
- setOperationAction(ISD::SMIN, MVT::v32i16, Legal);
- setOperationAction(ISD::UMIN, MVT::v64i8, Legal);
- setOperationAction(ISD::UMIN, MVT::v32i16, Legal);
-
setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v32i16, Custom);
setTruncStoreAction(MVT::v32i16, MVT::v32i8, Legal);
@@ -1579,6 +1515,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::MSTORE, VT, Legal);
setOperationAction(ISD::CTPOP, VT, Custom);
setOperationAction(ISD::CTTZ, VT, Custom);
+ setOperationAction(ISD::SMAX, VT, Legal);
+ setOperationAction(ISD::UMAX, VT, Legal);
+ setOperationAction(ISD::SMIN, VT, Legal);
+ setOperationAction(ISD::UMIN, VT, Legal);
setOperationPromotedToType(ISD::AND, VT, MVT::v8i64);
setOperationPromotedToType(ISD::OR, VT, MVT::v8i64);
@@ -1652,6 +1592,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::USUBO, VT, Custom);
setOperationAction(ISD::SMULO, VT, Custom);
setOperationAction(ISD::UMULO, VT, Custom);
+
+ // Support carry in as value rather than glue.
+ setOperationAction(ISD::ADDCARRY, VT, Custom);
+ setOperationAction(ISD::SUBCARRY, VT, Custom);
}
if (!Subtarget.is64Bit()) {
@@ -4839,14 +4783,10 @@ static bool isVEXTRACTIndex(SDNode *N, unsigned vecWidth) {
return false;
// The index should be aligned on a vecWidth-bit boundary.
- uint64_t Index =
- cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue();
-
+ uint64_t Index = N->getConstantOperandVal(1);
MVT VT = N->getSimpleValueType(0);
unsigned ElSize = VT.getScalarSizeInBits();
- bool Result = (Index * ElSize) % vecWidth == 0;
-
- return Result;
+ return (Index * ElSize) % vecWidth == 0;
}
/// Return true if the specified INSERT_SUBVECTOR
@@ -4856,15 +4796,12 @@ static bool isVINSERTIndex(SDNode *N, unsigned vecWidth) {
assert((vecWidth == 128 || vecWidth == 256) && "Unexpected vector width");
if (!isa<ConstantSDNode>(N->getOperand(2).getNode()))
return false;
- // The index should be aligned on a vecWidth-bit boundary.
- uint64_t Index =
- cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue();
+ // The index should be aligned on a vecWidth-bit boundary.
+ uint64_t Index = N->getConstantOperandVal(2);
MVT VT = N->getSimpleValueType(0);
unsigned ElSize = VT.getScalarSizeInBits();
- bool Result = (Index * ElSize) % vecWidth == 0;
-
- return Result;
+ return (Index * ElSize) % vecWidth == 0;
}
bool X86::isVINSERT128Index(SDNode *N) {
@@ -4888,13 +4825,9 @@ static unsigned getExtractVEXTRACTImmediate(SDNode *N, unsigned vecWidth) {
assert(isa<ConstantSDNode>(N->getOperand(1).getNode()) &&
"Illegal extract subvector for VEXTRACT");
- uint64_t Index =
- cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue();
-
+ uint64_t Index = N->getConstantOperandVal(1);
MVT VecVT = N->getOperand(0).getSimpleValueType();
- MVT ElVT = VecVT.getVectorElementType();
-
- unsigned NumElemsPerChunk = vecWidth / ElVT.getSizeInBits();
+ unsigned NumElemsPerChunk = vecWidth / VecVT.getScalarSizeInBits();
return Index / NumElemsPerChunk;
}
@@ -4903,13 +4836,9 @@ static unsigned getInsertVINSERTImmediate(SDNode *N, unsigned vecWidth) {
assert(isa<ConstantSDNode>(N->getOperand(2).getNode()) &&
"Illegal insert subvector for VINSERT");
- uint64_t Index =
- cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue();
-
+ uint64_t Index = N->getConstantOperandVal(2);
MVT VecVT = N->getSimpleValueType(0);
- MVT ElVT = VecVT.getVectorElementType();
-
- unsigned NumElemsPerChunk = vecWidth / ElVT.getSizeInBits();
+ unsigned NumElemsPerChunk = vecWidth / VecVT.getScalarSizeInBits();
return Index / NumElemsPerChunk;
}
@@ -4942,9 +4871,9 @@ bool X86::isZeroNode(SDValue Elt) {
return isNullConstant(Elt) || isNullFPConstant(Elt);
}
-// Build a vector of constants
+// Build a vector of constants.
// Use an UNDEF node if MaskElt == -1.
-// Spilt 64-bit constants in the 32-bit mode.
+// Split 64-bit constants in the 32-bit mode.
static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,
const SDLoc &dl, bool IsMask = false) {
@@ -5428,8 +5357,8 @@ static bool getTargetConstantBitsFromNode(SDValue Op, unsigned EltSizeInBits,
unsigned BitOffset) {
if (!Cst)
return false;
- unsigned CstSizeInBits = Cst->getType()->getPrimitiveSizeInBits();
if (isa<UndefValue>(Cst)) {
+ unsigned CstSizeInBits = Cst->getType()->getPrimitiveSizeInBits();
Undefs.setBits(BitOffset, BitOffset + CstSizeInBits);
return true;
}
@@ -6641,18 +6570,16 @@ static bool isUseOfShuffle(SDNode *N) {
return false;
}
-/// Attempt to use the vbroadcast instruction to generate a splat value for the
-/// following cases:
-/// 1. A splat BUILD_VECTOR which uses:
-/// a. A single scalar load, or a constant.
-/// b. Repeated pattern of constants (e.g. <0,1,0,1> or <0,1,2,3,0,1,2,3>).
-/// 2. A splat shuffle which uses a scalar_to_vector node which comes from
-/// a scalar load, or a constant.
+/// Attempt to use the vbroadcast instruction to generate a splat value
+/// from a splat BUILD_VECTOR which uses:
+/// a. A single scalar load, or a constant.
+/// b. Repeated pattern of constants (e.g. <0,1,0,1> or <0,1,2,3,0,1,2,3>).
///
/// The VBROADCAST node is returned when a pattern is found,
/// or SDValue() otherwise.
-static SDValue LowerVectorBroadcast(BuildVectorSDNode *BVOp, const X86Subtarget &Subtarget,
- SelectionDAG &DAG) {
+static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp,
+ const X86Subtarget &Subtarget,
+ SelectionDAG &DAG) {
// VBROADCAST requires AVX.
// TODO: Splats could be generated for non-AVX CPUs using SSE
// instructions, but there's less potential gain for only 128-bit vectors.
@@ -7605,7 +7532,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
return AddSub;
if (SDValue HorizontalOp = LowerToHorizontalOp(BV, Subtarget, DAG))
return HorizontalOp;
- if (SDValue Broadcast = LowerVectorBroadcast(BV, Subtarget, DAG))
+ if (SDValue Broadcast = lowerBuildVectorAsBroadcast(BV, Subtarget, DAG))
return Broadcast;
if (SDValue BitOp = lowerBuildVectorToBitOp(BV, DAG))
return BitOp;
@@ -9843,7 +9770,6 @@ static SDValue lowerVectorShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT,
/// For convenience, this code also bundles all of the subtarget feature set
/// filtering. While a little annoying to re-dispatch on type here, there isn't
/// a convenient way to factor it out.
-/// FIXME: This is very similar to LowerVectorBroadcast - can we merge them?
static SDValue lowerVectorShuffleAsBroadcast(const SDLoc &DL, MVT VT,
SDValue V1, SDValue V2,
ArrayRef<int> Mask,
@@ -16337,11 +16263,9 @@ SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC, const SDLoc &dl,
case ISD::ADD:
case ISD::SUB:
case ISD::MUL:
- case ISD::SHL: {
- const auto *BinNode = cast<BinaryWithFlagsSDNode>(Op.getNode());
- if (BinNode->Flags.hasNoSignedWrap())
+ case ISD::SHL:
+ if (Op.getNode()->getFlags().hasNoSignedWrap())
break;
- }
default:
NeedOF = true;
break;
@@ -16799,9 +16723,9 @@ static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC,
unsigned BitWidth = Op0.getValueSizeInBits();
unsigned AndBitWidth = And.getValueSizeInBits();
if (BitWidth > AndBitWidth) {
- APInt Zeros, Ones;
- DAG.computeKnownBits(Op0, Zeros, Ones);
- if (Zeros.countLeadingOnes() < BitWidth - AndBitWidth)
+ KnownBits Known;
+ DAG.computeKnownBits(Op0, Known);
+ if (Known.Zero.countLeadingOnes() < BitWidth - AndBitWidth)
return SDValue();
}
LHS = Op1;
@@ -23351,6 +23275,35 @@ static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
Op.getOperand(1), Op.getOperand(2));
}
+static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) {
+ SDNode *N = Op.getNode();
+ MVT VT = N->getSimpleValueType(0);
+
+ // Let legalize expand this if it isn't a legal type yet.
+ if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
+ return SDValue();
+
+ SDVTList VTs = DAG.getVTList(VT, MVT::i32);
+ SDLoc DL(N);
+
+ // Set the carry flag.
+ SDValue Carry = Op.getOperand(2);
+ EVT CarryVT = Carry.getValueType();
+ APInt NegOne = APInt::getAllOnesValue(CarryVT.getScalarSizeInBits());
+ Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
+ Carry, DAG.getConstant(NegOne, DL, CarryVT));
+
+ unsigned Opc = Op.getOpcode() == ISD::ADDCARRY ? X86ISD::ADC : X86ISD::SBB;
+ SDValue Sum = DAG.getNode(Opc, DL, VTs, Op.getOperand(0),
+ Op.getOperand(1), Carry.getValue(1));
+
+ SDValue SetCC = getSETCC(X86::COND_B, Sum.getValue(1), DL, DAG);
+ if (N->getValueType(1) == MVT::i1)
+ SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
+
+ return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
+}
+
static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
assert(Subtarget.isTargetDarwin() && Subtarget.is64Bit());
@@ -23862,6 +23815,8 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::ADDE:
case ISD::SUBC:
case ISD::SUBE: return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
+ case ISD::ADDCARRY:
+ case ISD::SUBCARRY: return LowerADDSUBCARRY(Op, DAG);
case ISD::ADD:
case ISD::SUB: return LowerADD_SUB(Op, DAG);
case ISD::SMAX:
@@ -26667,12 +26622,11 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
//===----------------------------------------------------------------------===//
void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
- APInt &KnownZero,
- APInt &KnownOne,
+ KnownBits &Known,
const APInt &DemandedElts,
const SelectionDAG &DAG,
unsigned Depth) const {
- unsigned BitWidth = KnownZero.getBitWidth();
+ unsigned BitWidth = Known.getBitWidth();
unsigned Opc = Op.getOpcode();
EVT VT = Op.getValueType();
assert((Opc >= ISD::BUILTIN_OP_END ||
@@ -26682,7 +26636,7 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
"Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!");
- KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything.
+ Known.Zero.clearAllBits(); Known.One.clearAllBits();
switch (Opc) {
default: break;
case X86ISD::ADD:
@@ -26701,33 +26655,33 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
break;
LLVM_FALLTHROUGH;
case X86ISD::SETCC:
- KnownZero.setBits(1, BitWidth);
+ Known.Zero.setBitsFrom(1);
break;
case X86ISD::MOVMSK: {
unsigned NumLoBits = Op.getOperand(0).getValueType().getVectorNumElements();
- KnownZero.setBits(NumLoBits, BitWidth);
+ Known.Zero.setBitsFrom(NumLoBits);
break;
}
case X86ISD::VSHLI:
case X86ISD::VSRLI: {
if (auto *ShiftImm = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
if (ShiftImm->getAPIntValue().uge(VT.getScalarSizeInBits())) {
- KnownZero = APInt::getAllOnesValue(BitWidth);
+ Known.Zero.setAllBits();
break;
}
- DAG.computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth + 1);
+ DAG.computeKnownBits(Op.getOperand(0), Known, Depth + 1);
unsigned ShAmt = ShiftImm->getZExtValue();
if (Opc == X86ISD::VSHLI) {
- KnownZero = KnownZero << ShAmt;
- KnownOne = KnownOne << ShAmt;
+ Known.Zero <<= ShAmt;
+ Known.One <<= ShAmt;
// Low bits are known zero.
- KnownZero.setLowBits(ShAmt);
+ Known.Zero.setLowBits(ShAmt);
} else {
- KnownZero.lshrInPlace(ShAmt);
- KnownOne.lshrInPlace(ShAmt);
+ Known.Zero.lshrInPlace(ShAmt);
+ Known.One.lshrInPlace(ShAmt);
// High bits are known zero.
- KnownZero.setHighBits(ShAmt);
+ Known.Zero.setHighBits(ShAmt);
}
}
break;
@@ -26741,12 +26695,12 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
unsigned InBitWidth = SrcVT.getScalarSizeInBits();
assert(InNumElts >= NumElts && "Illegal VZEXT input");
- KnownZero = KnownOne = APInt(InBitWidth, 0);
+ Known = KnownBits(InBitWidth);
APInt DemandedSrcElts = APInt::getLowBitsSet(InNumElts, NumElts);
- DAG.computeKnownBits(N0, KnownZero, KnownOne, DemandedSrcElts, Depth + 1);
- KnownOne = KnownOne.zext(BitWidth);
- KnownZero = KnownZero.zext(BitWidth);
- KnownZero.setBits(InBitWidth, BitWidth);
+ DAG.computeKnownBits(N0, Known, DemandedSrcElts, Depth + 1);
+ Known.One = Known.One.zext(BitWidth);
+ Known.Zero = Known.Zero.zext(BitWidth);
+ Known.Zero.setBitsFrom(InBitWidth);
break;
}
}
@@ -30206,12 +30160,11 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
assert(BitWidth >= 8 && BitWidth <= 64 && "Invalid mask size");
APInt DemandedMask(APInt::getSignMask(BitWidth));
- APInt KnownZero, KnownOne;
+ KnownBits Known;
TargetLowering::TargetLoweringOpt TLO(DAG, DCI.isBeforeLegalize(),
DCI.isBeforeLegalizeOps());
if (TLI.ShrinkDemandedConstant(Cond, DemandedMask, TLO) ||
- TLI.SimplifyDemandedBits(Cond, DemandedMask, KnownZero, KnownOne,
- TLO)) {
+ TLI.SimplifyDemandedBits(Cond, DemandedMask, Known, TLO)) {
// If we changed the computation somewhere in the DAG, this change will
// affect all users of Cond. Make sure it is fine and update all the nodes
// so that we do not use the generic VSELECT anymore. Otherwise, we may
@@ -31056,8 +31009,7 @@ static SDValue combineShiftLeft(SDNode *N, SelectionDAG &DAG) {
N0.getOperand(1).getOpcode() == ISD::Constant) {
SDValue N00 = N0.getOperand(0);
APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
- const APInt &ShAmt = N1C->getAPIntValue();
- Mask = Mask.shl(ShAmt);
+ Mask <<= N1C->getAPIntValue();
bool MaskOK = false;
// We can handle cases concerning bit-widening nodes containing setcc_c if
// we carefully interrogate the mask to make sure we are semantics
@@ -31267,9 +31219,9 @@ static SDValue combineVectorShiftImm(SDNode *N, SelectionDAG &DAG,
unsigned ShiftImm = ShiftVal.getZExtValue();
for (APInt &Elt : EltBits) {
if (X86ISD::VSHLI == Opcode)
- Elt = Elt.shl(ShiftImm);
+ Elt <<= ShiftImm;
else if (X86ISD::VSRAI == Opcode)
- Elt = Elt.ashr(ShiftImm);
+ Elt.ashrInPlace(ShiftImm);
else
Elt.lshrInPlace(ShiftImm);
}
@@ -33481,7 +33433,7 @@ static SDValue combineFneg(SDNode *N, SelectionDAG &DAG,
// use of a constant by performing (-0 - A*B) instead.
// FIXME: Check rounding control flags as well once it becomes available.
if (Arg.getOpcode() == ISD::FMUL && (SVT == MVT::f32 || SVT == MVT::f64) &&
- Arg->getFlags()->hasNoSignedZeros() && Subtarget.hasAnyFMA()) {
+ Arg->getFlags().hasNoSignedZeros() && Subtarget.hasAnyFMA()) {
SDValue Zero = DAG.getConstantFP(0.0, DL, VT);
SDValue NewNode = DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0),
Arg.getOperand(1), Zero);
@@ -33775,12 +33727,12 @@ static SDValue combineBT(SDNode *N, SelectionDAG &DAG,
if (Op1.hasOneUse()) {
unsigned BitWidth = Op1.getValueSizeInBits();
APInt DemandedMask = APInt::getLowBitsSet(BitWidth, Log2_32(BitWidth));
- APInt KnownZero, KnownOne;
+ KnownBits Known;
TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
!DCI.isBeforeLegalizeOps());
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
if (TLI.ShrinkDemandedConstant(Op1, DemandedMask, TLO) ||
- TLI.SimplifyDemandedBits(Op1, DemandedMask, KnownZero, KnownOne, TLO))
+ TLI.SimplifyDemandedBits(Op1, DemandedMask, Known, TLO))
DCI.CommitTargetLoweringOpt(TLO);
}
return SDValue();
@@ -33842,8 +33794,8 @@ static SDValue promoteExtBeforeAdd(SDNode *Ext, SelectionDAG &DAG,
return SDValue();
bool Sext = Ext->getOpcode() == ISD::SIGN_EXTEND;
- bool NSW = Add->getFlags()->hasNoSignedWrap();
- bool NUW = Add->getFlags()->hasNoUnsignedWrap();
+ bool NSW = Add->getFlags().hasNoSignedWrap();
+ bool NUW = Add->getFlags().hasNoUnsignedWrap();
// We need an 'add nsw' feeding into the 'sext' or 'add nuw' feeding
// into the 'zext'
@@ -33883,7 +33835,7 @@ static SDValue promoteExtBeforeAdd(SDNode *Ext, SelectionDAG &DAG,
SDNodeFlags Flags;
Flags.setNoSignedWrap(NSW);
Flags.setNoUnsignedWrap(NUW);
- return DAG.getNode(ISD::ADD, SDLoc(Add), VT, NewExt, NewConstant, &Flags);
+ return DAG.getNode(ISD::ADD, SDLoc(Add), VT, NewExt, NewConstant, Flags);
}
/// (i8,i32 {s/z}ext ({s/u}divrem (i8 x, i8 y)) ->
@@ -34486,6 +34438,34 @@ static SDValue combineSIntToFP(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
+// Optimize RES, EFLAGS = X86ISD::ADD LHS, RHS
+static SDValue combineX86ADD(SDNode *N, SelectionDAG &DAG,
+ X86TargetLowering::DAGCombinerInfo &DCI) {
+ // When legalizing carry, we create carries via add X, -1
+ // If that comes from an actual carry, via setcc, we use the
+ // carry directly.
+ if (isAllOnesConstant(N->getOperand(1)) && N->hasAnyUseOfValue(1)) {
+ SDValue Carry = N->getOperand(0);
+ while (Carry.getOpcode() == ISD::TRUNCATE ||
+ Carry.getOpcode() == ISD::ZERO_EXTEND ||
+ Carry.getOpcode() == ISD::SIGN_EXTEND ||
+ Carry.getOpcode() == ISD::ANY_EXTEND ||
+ (Carry.getOpcode() == ISD::AND &&
+ isOneConstant(Carry.getOperand(1))))
+ Carry = Carry.getOperand(0);
+
+ if (Carry.getOpcode() == ISD::SETCC ||
+ Carry.getOpcode() == X86ISD::SETCC ||
+ Carry.getOpcode() == X86ISD::SETCC_CARRY) {
+ auto *Cond = cast<ConstantSDNode>(Carry.getOperand(0));
+ if (Cond->getZExtValue() == X86::COND_B)
+ return DCI.CombineTo(N, SDValue(N, 0), Carry.getOperand(1));
+ }
+ }
+
+ return SDValue();
+}
+
// Optimize RES, EFLAGS = X86ISD::ADC LHS, RHS, EFLAGS
static SDValue combineADC(SDNode *N, SelectionDAG &DAG,
X86TargetLowering::DAGCombinerInfo &DCI) {
@@ -34740,8 +34720,8 @@ static SDValue combineLoopSADPattern(SDNode *N, SelectionDAG &DAG,
static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
- const SDNodeFlags *Flags = &cast<BinaryWithFlagsSDNode>(N)->Flags;
- if (Flags->hasVectorReduction()) {
+ const SDNodeFlags Flags = N->getFlags();
+ if (Flags.hasVectorReduction()) {
if (SDValue Sad = combineLoopSADPattern(N, DAG, Subtarget))
return Sad;
if (SDValue MAdd = combineLoopMAddPattern(N, DAG, Subtarget))
@@ -35047,6 +35027,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
case X86ISD::CMOV: return combineCMov(N, DAG, DCI, Subtarget);
case ISD::ADD: return combineAdd(N, DAG, Subtarget);
case ISD::SUB: return combineSub(N, DAG, Subtarget);
+ case X86ISD::ADD: return combineX86ADD(N, DAG, DCI);
case X86ISD::ADC: return combineADC(N, DAG, DCI);
case ISD::MUL: return combineMul(N, DAG, DCI, Subtarget);
case ISD::SHL:
@@ -35171,14 +35152,21 @@ bool X86TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const {
/// know that the code that lowers COPY of EFLAGS has to use the stack, and if
/// we don't adjust the stack we clobber the first frame index.
/// See X86InstrInfo::copyPhysReg.
-bool X86TargetLowering::hasCopyImplyingStackAdjustment(
- MachineFunction *MF) const {
- const MachineRegisterInfo &MRI = MF->getRegInfo();
-
+static bool hasCopyImplyingStackAdjustment(const MachineFunction &MF) {
+ const MachineRegisterInfo &MRI = MF.getRegInfo();
return any_of(MRI.reg_instructions(X86::EFLAGS),
[](const MachineInstr &RI) { return RI.isCopy(); });
}
+void X86TargetLowering::finalizeLowering(MachineFunction &MF) const {
+ if (hasCopyImplyingStackAdjustment(MF)) {
+ MachineFrameInfo &MFI = MF.getFrameInfo();
+ MFI.setHasCopyImplyingStackAdjustment(true);
+ }
+
+ TargetLoweringBase::finalizeLowering(MF);
+}
+
/// This method query the target whether it is beneficial for dag combiner to
/// promote the specified node. If true, it should return the desired promotion
/// type by reference.
diff --git a/lib/Target/X86/X86ISelLowering.h b/lib/Target/X86/X86ISelLowering.h
index 190a883350000..46dc587c637d7 100644
--- a/lib/Target/X86/X86ISelLowering.h
+++ b/lib/Target/X86/X86ISelLowering.h
@@ -773,10 +773,6 @@ namespace llvm {
/// and some i16 instructions are slow.
bool IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const override;
- /// Return true if the MachineFunction contains a COPY which would imply
- /// HasOpaqueSPAdjustment.
- bool hasCopyImplyingStackAdjustment(MachineFunction *MF) const override;
-
MachineBasicBlock *
EmitInstrWithCustomInserter(MachineInstr &MI,
MachineBasicBlock *MBB) const override;
@@ -828,8 +824,7 @@ namespace llvm {
/// Determine which of the bits specified in Mask are known to be either
/// zero or one and return them in the KnownZero/KnownOne bitsets.
void computeKnownBitsForTargetNode(const SDValue Op,
- APInt &KnownZero,
- APInt &KnownOne,
+ KnownBits &Known,
const APInt &DemandedElts,
const SelectionDAG &DAG,
unsigned Depth = 0) const override;
@@ -1066,6 +1061,9 @@ namespace llvm {
ArrayRef<ShuffleVectorInst *> Shuffles,
ArrayRef<unsigned> Indices,
unsigned Factor) const override;
+
+ void finalizeLowering(MachineFunction &MF) const override;
+
protected:
std::pair<const TargetRegisterClass *, uint8_t>
findRepresentativeClass(const TargetRegisterInfo *TRI,
diff --git a/lib/Target/X86/X86InstrCompiler.td b/lib/Target/X86/X86InstrCompiler.td
index e592c2b3c0aa5..3dc673e3c35a4 100644
--- a/lib/Target/X86/X86InstrCompiler.td
+++ b/lib/Target/X86/X86InstrCompiler.td
@@ -1271,11 +1271,11 @@ def or_is_add : PatFrag<(ops node:$lhs, node:$rhs), (or node:$lhs, node:$rhs),[{
if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
return CurDAG->MaskedValueIsZero(N->getOperand(0), CN->getAPIntValue());
- APInt KnownZero0, KnownOne0;
- CurDAG->computeKnownBits(N->getOperand(0), KnownZero0, KnownOne0, 0);
- APInt KnownZero1, KnownOne1;
- CurDAG->computeKnownBits(N->getOperand(1), KnownZero1, KnownOne1, 0);
- return (~KnownZero0 & ~KnownZero1) == 0;
+ KnownBits Known0;
+ CurDAG->computeKnownBits(N->getOperand(0), Known0, 0);
+ KnownBits Known1;
+ CurDAG->computeKnownBits(N->getOperand(1), Known1, 0);
+ return (~Known0.Zero & ~Known1.Zero) == 0;
}]>;
diff --git a/lib/Target/X86/X86InstrInfo.td b/lib/Target/X86/X86InstrInfo.td
index c3def461afdcc..ce0876498677e 100644
--- a/lib/Target/X86/X86InstrInfo.td
+++ b/lib/Target/X86/X86InstrInfo.td
@@ -877,7 +877,9 @@ def In32BitMode : Predicate<"Subtarget->is32Bit()">,
def IsWin64 : Predicate<"Subtarget->isTargetWin64()">;
def NotWin64 : Predicate<"!Subtarget->isTargetWin64()">;
def NotWin64WithoutFP : Predicate<"!Subtarget->isTargetWin64() ||"
- "Subtarget->getFrameLowering()->hasFP(*MF)">;
+ "Subtarget->getFrameLowering()->hasFP(*MF)"> {
+ let RecomputePerFunction = 1;
+}
def IsPS4 : Predicate<"Subtarget->isTargetPS4()">;
def NotPS4 : Predicate<"!Subtarget->isTargetPS4()">;
def IsNaCl : Predicate<"Subtarget->isTargetNaCl()">;
@@ -887,9 +889,9 @@ def KernelCode : Predicate<"TM.getCodeModel() == CodeModel::Kernel">;
def NearData : Predicate<"TM.getCodeModel() == CodeModel::Small ||"
"TM.getCodeModel() == CodeModel::Kernel">;
def IsNotPIC : Predicate<"!TM.isPositionIndependent()">;
-def OptForSize : Predicate<"OptForSize">;
-def OptForMinSize : Predicate<"OptForMinSize">;
-def OptForSpeed : Predicate<"!OptForSize">;
+def OptForSize : Predicate<"Subtarget->getOptForSize()">;
+def OptForMinSize : Predicate<"Subtarget->getOptForMinSize()">;
+def OptForSpeed : Predicate<"!Subtarget->getOptForSize()">;
def FastBTMem : Predicate<"!Subtarget->isBTMemSlow()">;
def CallImmAddr : Predicate<"Subtarget->isLegalToCallImmediateAddr()">;
def FavorMemIndirectCall : Predicate<"!Subtarget->callRegIndirect()">;
diff --git a/lib/Target/X86/X86InstructionSelector.cpp b/lib/Target/X86/X86InstructionSelector.cpp
index d0f1b7091da95..38f7bc0af5c72 100644
--- a/lib/Target/X86/X86InstructionSelector.cpp
+++ b/lib/Target/X86/X86InstructionSelector.cpp
@@ -48,7 +48,6 @@ public:
X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI,
const X86RegisterBankInfo &RBI);
- void beginFunction(const MachineFunction &MF) override;
bool select(MachineInstr &I) const override;
private:
@@ -56,11 +55,9 @@ private:
/// the patterns that don't require complex C++.
bool selectImpl(MachineInstr &I) const;
- // TODO: remove after selectImpl support pattern with a predicate.
+ // TODO: remove after suported by Tablegen-erated instruction selection.
unsigned getFAddOp(LLT &Ty, const RegisterBank &RB) const;
unsigned getFSubOp(LLT &Ty, const RegisterBank &RB) const;
- unsigned getAddOp(LLT &Ty, const RegisterBank &RB) const;
- unsigned getSubOp(LLT &Ty, const RegisterBank &RB) const;
unsigned getLoadStoreOp(LLT &Ty, const RegisterBank &RB, unsigned Opc,
uint64_t Alignment) const;
@@ -80,12 +77,10 @@ private:
const X86InstrInfo &TII;
const X86RegisterInfo &TRI;
const X86RegisterBankInfo &RBI;
- bool OptForSize;
- bool OptForMinSize;
- PredicateBitset AvailableFeatures;
- PredicateBitset computeAvailableFeatures(const MachineFunction *MF,
- const X86Subtarget *Subtarget) const;
+#define GET_GLOBALISEL_PREDICATES_DECL
+#include "X86GenGlobalISel.inc"
+#undef GET_GLOBALISEL_PREDICATES_DECL
#define GET_GLOBALISEL_TEMPORARIES_DECL
#include "X86GenGlobalISel.inc"
@@ -102,8 +97,10 @@ X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM,
const X86Subtarget &STI,
const X86RegisterBankInfo &RBI)
: InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
- TRI(*STI.getRegisterInfo()), RBI(RBI), OptForSize(false),
- OptForMinSize(false), AvailableFeatures()
+ TRI(*STI.getRegisterInfo()), RBI(RBI),
+#define GET_GLOBALISEL_PREDICATES_INIT
+#include "X86GenGlobalISel.inc"
+#undef GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
#include "X86GenGlobalISel.inc"
#undef GET_GLOBALISEL_TEMPORARIES_INIT
@@ -153,10 +150,9 @@ static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
- (void)DstSize;
unsigned SrcReg = I.getOperand(1).getReg();
const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
- (void)SrcSize;
+
assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) &&
"No phys reg on generic operators");
assert((DstSize == SrcSize ||
@@ -172,6 +168,18 @@ static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
case X86::GPRRegBankID:
assert((DstSize <= 64) && "GPRs cannot get more than 64-bit width values.");
RC = getRegClassForTypeOnBank(MRI.getType(DstReg), RegBank);
+
+ // Change the physical register
+ if (SrcSize > DstSize && TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
+ if (RC == &X86::GR32RegClass)
+ I.getOperand(1).setSubReg(X86::sub_32bit);
+ else if (RC == &X86::GR16RegClass)
+ I.getOperand(1).setSubReg(X86::sub_16bit);
+ else if (RC == &X86::GR8RegClass)
+ I.getOperand(1).setSubReg(X86::sub_8bit);
+
+ I.getOperand(1).substPhysReg(SrcReg, TRI);
+ }
break;
case X86::VECRRegBankID:
RC = getRegClassForTypeOnBank(MRI.getType(DstReg), RegBank);
@@ -195,12 +203,6 @@ static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
return true;
}
-void X86InstructionSelector::beginFunction(const MachineFunction &MF) {
- OptForSize = MF.getFunction()->optForSize();
- OptForMinSize = MF.getFunction()->optForMinSize();
- AvailableFeatures = computeAvailableFeatures(&MF, &STI);
-}
-
bool X86InstructionSelector::select(MachineInstr &I) const {
assert(I.getParent() && "Instruction should be in a basic block!");
assert(I.getParent()->getParent() && "Instruction should be in a function!");
@@ -223,8 +225,12 @@ bool X86InstructionSelector::select(MachineInstr &I) const {
assert(I.getNumOperands() == I.getNumExplicitOperands() &&
"Generic instruction has unexpected implicit operands\n");
- // TODO: This should be implemented by tblgen, pattern with predicate not
- // supported yet.
+ if (selectImpl(I))
+ return true;
+
+ DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs()));
+
+ // TODO: This should be implemented by tblgen.
if (selectBinaryOp(I, MRI, MF))
return true;
if (selectLoadStoreOp(I, MRI, MF))
@@ -236,7 +242,7 @@ bool X86InstructionSelector::select(MachineInstr &I) const {
if (selectTrunc(I, MRI, MF))
return true;
- return selectImpl(I);
+ return false;
}
unsigned X86InstructionSelector::getFAddOp(LLT &Ty,
@@ -309,44 +315,6 @@ unsigned X86InstructionSelector::getFSubOp(LLT &Ty,
return TargetOpcode::G_FSUB;
}
-unsigned X86InstructionSelector::getAddOp(LLT &Ty,
- const RegisterBank &RB) const {
-
- if (X86::VECRRegBankID != RB.getID())
- return TargetOpcode::G_ADD;
-
- if (Ty == LLT::vector(4, 32)) {
- if (STI.hasAVX512() && STI.hasVLX()) {
- return X86::VPADDDZ128rr;
- } else if (STI.hasAVX()) {
- return X86::VPADDDrr;
- } else if (STI.hasSSE2()) {
- return X86::PADDDrr;
- }
- }
-
- return TargetOpcode::G_ADD;
-}
-
-unsigned X86InstructionSelector::getSubOp(LLT &Ty,
- const RegisterBank &RB) const {
-
- if (X86::VECRRegBankID != RB.getID())
- return TargetOpcode::G_SUB;
-
- if (Ty == LLT::vector(4, 32)) {
- if (STI.hasAVX512() && STI.hasVLX()) {
- return X86::VPSUBDZ128rr;
- } else if (STI.hasAVX()) {
- return X86::VPSUBDrr;
- } else if (STI.hasSSE2()) {
- return X86::PSUBDrr;
- }
- }
-
- return TargetOpcode::G_SUB;
-}
-
bool X86InstructionSelector::selectBinaryOp(MachineInstr &I,
MachineRegisterInfo &MRI,
MachineFunction &MF) const {
@@ -364,12 +332,6 @@ bool X86InstructionSelector::selectBinaryOp(MachineInstr &I,
case TargetOpcode::G_FSUB:
NewOpc = getFSubOp(Ty, RB);
break;
- case TargetOpcode::G_ADD:
- NewOpc = getAddOp(Ty, RB);
- break;
- case TargetOpcode::G_SUB:
- NewOpc = getSubOp(Ty, RB);
- break;
default:
break;
}
@@ -396,7 +358,7 @@ unsigned X86InstructionSelector::getLoadStoreOp(LLT &Ty, const RegisterBank &RB,
} else if (Ty == LLT::scalar(16)) {
if (X86::GPRRegBankID == RB.getID())
return Isload ? X86::MOV16rm : X86::MOV16mr;
- } else if (Ty == LLT::scalar(32)) {
+ } else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) {
if (X86::GPRRegBankID == RB.getID())
return Isload ? X86::MOV32rm : X86::MOV32mr;
if (X86::VECRRegBankID == RB.getID())
@@ -404,7 +366,7 @@ unsigned X86InstructionSelector::getLoadStoreOp(LLT &Ty, const RegisterBank &RB,
: HasAVX ? X86::VMOVSSrm : X86::MOVSSrm)
: (HasAVX512 ? X86::VMOVSSZmr
: HasAVX ? X86::VMOVSSmr : X86::MOVSSmr);
- } else if (Ty == LLT::scalar(64)) {
+ } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
if (X86::GPRRegBankID == RB.getID())
return Isload ? X86::MOV64rm : X86::MOV64mr;
if (X86::VECRRegBankID == RB.getID())
diff --git a/lib/Target/X86/X86LegalizerInfo.cpp b/lib/Target/X86/X86LegalizerInfo.cpp
index c2dc762fec5eb..a437f6bf4714d 100644
--- a/lib/Target/X86/X86LegalizerInfo.cpp
+++ b/lib/Target/X86/X86LegalizerInfo.cpp
@@ -71,6 +71,15 @@ void X86LegalizerInfo::setLegalizerInfo32bit() {
setAction({TargetOpcode::G_CONSTANT, s1}, WidenScalar);
setAction({TargetOpcode::G_CONSTANT, s64}, NarrowScalar);
+
+ // Extensions
+ setAction({G_ZEXT, s32}, Legal);
+ setAction({G_SEXT, s32}, Legal);
+
+ for (auto Ty : {s8, s16}) {
+ setAction({G_ZEXT, 1, Ty}, Legal);
+ setAction({G_SEXT, 1, Ty}, Legal);
+ }
}
void X86LegalizerInfo::setLegalizerInfo64bit() {
@@ -105,6 +114,17 @@ void X86LegalizerInfo::setLegalizerInfo64bit() {
setAction({TargetOpcode::G_CONSTANT, Ty}, Legal);
setAction({TargetOpcode::G_CONSTANT, s1}, WidenScalar);
+
+ // Extensions
+ for (auto Ty : {s32, s64}) {
+ setAction({G_ZEXT, Ty}, Legal);
+ setAction({G_SEXT, Ty}, Legal);
+ }
+
+ for (auto Ty : {s8, s16, s32}) {
+ setAction({G_ZEXT, 1, Ty}, Legal);
+ setAction({G_SEXT, 1, Ty}, Legal);
+ }
}
void X86LegalizerInfo::setLegalizerInfoSSE1() {
diff --git a/lib/Target/X86/X86OptimizeLEAs.cpp b/lib/Target/X86/X86OptimizeLEAs.cpp
index debb192732e5c..7be0a7fd40678 100644
--- a/lib/Target/X86/X86OptimizeLEAs.cpp
+++ b/lib/Target/X86/X86OptimizeLEAs.cpp
@@ -27,6 +27,8 @@
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/Passes.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/DIBuilder.h"
#include "llvm/IR/Function.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
@@ -221,6 +223,8 @@ public:
StringRef getPassName() const override { return "X86 LEA Optimize"; }
+ bool doInitialization(Module &M) override;
+
/// \brief Loop over all of the basic blocks, replacing address
/// calculations in load and store instructions, if it's already
/// been calculated by LEA. Also, remove redundant LEAs.
@@ -262,6 +266,12 @@ private:
/// \brief Removes redundant address calculations.
bool removeRedundantAddrCalc(MemOpMap &LEAs);
+ /// Replace debug value MI with a new debug value instruction using register
+ /// VReg with an appropriate offset and DIExpression to incorporate the
+ /// address displacement AddrDispShift. Return new debug value instruction.
+ MachineInstr *replaceDebugValue(MachineInstr &MI, unsigned VReg,
+ int64_t AddrDispShift);
+
/// \brief Removes LEAs which calculate similar addresses.
bool removeRedundantLEAs(MemOpMap &LEAs);
@@ -270,6 +280,7 @@ private:
MachineRegisterInfo *MRI;
const X86InstrInfo *TII;
const X86RegisterInfo *TRI;
+ Module *TheModule;
static char ID;
};
@@ -532,6 +543,25 @@ bool OptimizeLEAPass::removeRedundantAddrCalc(MemOpMap &LEAs) {
return Changed;
}
+MachineInstr *OptimizeLEAPass::replaceDebugValue(MachineInstr &MI,
+ unsigned VReg,
+ int64_t AddrDispShift) {
+ DIExpression *Expr = const_cast<DIExpression *>(MI.getDebugExpression());
+
+ if (AddrDispShift != 0)
+ Expr = DIExpression::prepend(Expr, DIExpression::NoDeref, AddrDispShift,
+ DIExpression::WithStackValue);
+
+ // Replace DBG_VALUE instruction with modified version.
+ MachineBasicBlock *MBB = MI.getParent();
+ DebugLoc DL = MI.getDebugLoc();
+ bool IsIndirect = MI.isIndirectDebugValue();
+ int64_t Offset = IsIndirect ? MI.getOperand(1).getImm() : 0;
+ const MDNode *Var = MI.getDebugVariable();
+ return BuildMI(*MBB, MBB->erase(&MI), DL, TII->get(TargetOpcode::DBG_VALUE),
+ IsIndirect, VReg, Offset, Var, Expr);
+}
+
// Try to find similar LEAs in the list and replace one with another.
bool OptimizeLEAPass::removeRedundantLEAs(MemOpMap &LEAs) {
bool Changed = false;
@@ -563,13 +593,21 @@ bool OptimizeLEAPass::removeRedundantLEAs(MemOpMap &LEAs) {
// Loop over all uses of the Last LEA and update their operands. Note
// that the correctness of this has already been checked in the
// isReplaceable function.
+ unsigned FirstVReg = First.getOperand(0).getReg();
unsigned LastVReg = Last.getOperand(0).getReg();
- for (auto UI = MRI->use_nodbg_begin(LastVReg),
- UE = MRI->use_nodbg_end();
+ for (auto UI = MRI->use_begin(LastVReg), UE = MRI->use_end();
UI != UE;) {
MachineOperand &MO = *UI++;
MachineInstr &MI = *MO.getParent();
+ if (MI.isDebugValue()) {
+ // Replace DBG_VALUE instruction with modified version using the
+ // register from the replacing LEA and the address displacement
+ // between the LEA instructions.
+ replaceDebugValue(MI, FirstVReg, AddrDispShift);
+ continue;
+ }
+
// Get the number of the first memory operand.
const MCInstrDesc &Desc = MI.getDesc();
int MemOpNo =
@@ -577,7 +615,7 @@ bool OptimizeLEAPass::removeRedundantLEAs(MemOpMap &LEAs) {
X86II::getOperandBias(Desc);
// Update address base.
- MO.setReg(First.getOperand(0).getReg());
+ MO.setReg(FirstVReg);
// Update address disp.
MachineOperand &Op = MI.getOperand(MemOpNo + X86::AddrDisp);
@@ -587,11 +625,8 @@ bool OptimizeLEAPass::removeRedundantLEAs(MemOpMap &LEAs) {
Op.setOffset(Op.getOffset() + AddrDispShift);
}
- // Mark debug values referring to Last LEA as undefined.
- MRI->markUsesInDebugValueAsUndef(LastVReg);
-
// Since we can possibly extend register lifetime, clear kill flags.
- MRI->clearKillFlags(First.getOperand(0).getReg());
+ MRI->clearKillFlags(FirstVReg);
++NumRedundantLEAs;
DEBUG(dbgs() << "OptimizeLEAs: Remove redundant LEA: "; Last.dump(););
@@ -614,6 +649,11 @@ bool OptimizeLEAPass::removeRedundantLEAs(MemOpMap &LEAs) {
return Changed;
}
+bool OptimizeLEAPass::doInitialization(Module &M) {
+ TheModule = &M;
+ return false;
+}
+
bool OptimizeLEAPass::runOnMachineFunction(MachineFunction &MF) {
bool Changed = false;
diff --git a/lib/Target/X86/X86SelectionDAGInfo.cpp b/lib/Target/X86/X86SelectionDAGInfo.cpp
index 1a72a0ba3a642..d4b2392eb1f52 100644
--- a/lib/Target/X86/X86SelectionDAGInfo.cpp
+++ b/lib/Target/X86/X86SelectionDAGInfo.cpp
@@ -44,8 +44,26 @@ bool X86SelectionDAGInfo::isBaseRegConflictPossible(
return false;
}
+namespace {
+
+// Represents a cover of a buffer of Size bytes with Count() blocks of type AVT
+// (of size UBytes() bytes), as well as how many bytes remain (BytesLeft() is
+// always smaller than the block size).
+struct RepMovsRepeats {
+ RepMovsRepeats(uint64_t Size) : Size(Size) {}
+
+ uint64_t Count() const { return Size / UBytes(); }
+ uint64_t BytesLeft() const { return Size % UBytes(); }
+ uint64_t UBytes() const { return AVT.getSizeInBits() / 8; }
+
+ const uint64_t Size;
+ MVT AVT = MVT::i8;
+};
+
+} // namespace
+
SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
- SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
+ SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Val,
SDValue Size, unsigned Align, bool isVolatile,
MachinePointerInfo DstPtrInfo) const {
ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
@@ -69,10 +87,10 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
if ((Align & 3) != 0 || !ConstantSize ||
ConstantSize->getZExtValue() > Subtarget.getMaxInlineSizeThreshold()) {
// Check to see if there is a specialized entry-point for memory zeroing.
- ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src);
+ ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Val);
- if (const char *bzeroEntry = V &&
- V->isNullValue() ? Subtarget.getBZeroEntry() : nullptr) {
+ if (const char *bzeroEntry = ValC &&
+ ValC->isNullValue() ? Subtarget.getBZeroEntry() : nullptr) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
@@ -104,7 +122,7 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
SDValue InFlag;
EVT AVT;
SDValue Count;
- ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Src);
+ ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Val);
unsigned BytesLeft = 0;
if (ValC) {
unsigned ValReg;
@@ -147,7 +165,7 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
} else {
AVT = MVT::i8;
Count = DAG.getIntPtrConstant(SizeVal, dl);
- Chain = DAG.getCopyToReg(Chain, dl, X86::AL, Src, InFlag);
+ Chain = DAG.getCopyToReg(Chain, dl, X86::AL, Val, InFlag);
InFlag = Chain.getValue(1);
}
@@ -171,7 +189,7 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
Chain = DAG.getMemset(Chain, dl,
DAG.getNode(ISD::ADD, dl, AddrVT, Dst,
DAG.getConstant(Offset, dl, AddrVT)),
- Src,
+ Val,
DAG.getConstant(BytesLeft, dl, SizeVT),
Align, isVolatile, false,
DstPtrInfo.getWithOffset(Offset));
@@ -181,24 +199,6 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
return Chain;
}
-namespace {
-
-// Represents a cover of a buffer of SizeVal bytes with blocks of size
-// AVT, as well as how many bytes remain (BytesLeft is always smaller than
-// the block size).
-struct RepMovsRepeats {
- RepMovsRepeats(const uint64_t SizeVal, const MVT& AVT) {
- const unsigned UBytes = AVT.getSizeInBits() / 8;
- Count = SizeVal / UBytes;
- BytesLeft = SizeVal % UBytes;
- }
-
- unsigned Count;
- unsigned BytesLeft;
-};
-
-} // namespace
-
SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy(
SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
SDValue Size, unsigned Align, bool isVolatile, bool AlwaysInline,
@@ -210,8 +210,8 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy(
DAG.getMachineFunction().getSubtarget<X86Subtarget>();
if (!ConstantSize)
return SDValue();
- uint64_t SizeVal = ConstantSize->getZExtValue();
- if (!AlwaysInline && SizeVal > Subtarget.getMaxInlineSizeThreshold())
+ RepMovsRepeats Repeats(ConstantSize->getZExtValue());
+ if (!AlwaysInline && Repeats.Size > Subtarget.getMaxInlineSizeThreshold())
return SDValue();
/// If not DWORD aligned, it is more efficient to call the library. However
@@ -232,35 +232,31 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy(
if (isBaseRegConflictPossible(DAG, ClobberSet))
return SDValue();
- MVT AVT;
- if (Subtarget.hasERMSB())
- // If the target has enhanced REPMOVSB, then it's at least as fast to use
- // REP MOVSB instead of REP MOVS{W,D,Q}, and it avoids having to handle
- // BytesLeft.
- AVT = MVT::i8;
- else if (Align & 1)
- AVT = MVT::i8;
- else if (Align & 2)
- AVT = MVT::i16;
- else if (Align & 4)
- // DWORD aligned
- AVT = MVT::i32;
- else
- // QWORD aligned
- AVT = Subtarget.is64Bit() ? MVT::i64 : MVT::i32;
+ // If the target has enhanced REPMOVSB, then it's at least as fast to use
+ // REP MOVSB instead of REP MOVS{W,D,Q}, and it avoids having to handle
+ // BytesLeft.
+ if (!Subtarget.hasERMSB() && !(Align & 1)) {
+ if (Align & 2)
+ // WORD aligned
+ Repeats.AVT = MVT::i16;
+ else if (Align & 4)
+ // DWORD aligned
+ Repeats.AVT = MVT::i32;
+ else
+ // QWORD aligned
+ Repeats.AVT = Subtarget.is64Bit() ? MVT::i64 : MVT::i32;
- RepMovsRepeats Repeats(SizeVal, AVT);
- if (Repeats.BytesLeft > 0 &&
- DAG.getMachineFunction().getFunction()->optForMinSize()) {
- // When agressively optimizing for size, avoid generating the code to handle
- // BytesLeft.
- AVT = MVT::i8;
- Repeats = RepMovsRepeats(SizeVal, AVT);
+ if (Repeats.BytesLeft() > 0 &&
+ DAG.getMachineFunction().getFunction()->optForMinSize()) {
+ // When agressively optimizing for size, avoid generating the code to
+ // handle BytesLeft.
+ Repeats.AVT = MVT::i8;
+ }
}
SDValue InFlag;
Chain = DAG.getCopyToReg(Chain, dl, Subtarget.is64Bit() ? X86::RCX : X86::ECX,
- DAG.getIntPtrConstant(Repeats.Count, dl), InFlag);
+ DAG.getIntPtrConstant(Repeats.Count(), dl), InFlag);
InFlag = Chain.getValue(1);
Chain = DAG.getCopyToReg(Chain, dl, Subtarget.is64Bit() ? X86::RDI : X86::EDI,
Dst, InFlag);
@@ -270,14 +266,14 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy(
InFlag = Chain.getValue(1);
SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
- SDValue Ops[] = { Chain, DAG.getValueType(AVT), InFlag };
+ SDValue Ops[] = { Chain, DAG.getValueType(Repeats.AVT), InFlag };
SDValue RepMovs = DAG.getNode(X86ISD::REP_MOVS, dl, Tys, Ops);
SmallVector<SDValue, 4> Results;
Results.push_back(RepMovs);
- if (Repeats.BytesLeft) {
+ if (Repeats.BytesLeft()) {
// Handle the last 1 - 7 bytes.
- unsigned Offset = SizeVal - Repeats.BytesLeft;
+ unsigned Offset = Repeats.Size - Repeats.BytesLeft();
EVT DstVT = Dst.getValueType();
EVT SrcVT = Src.getValueType();
EVT SizeVT = Size.getValueType();
@@ -288,7 +284,7 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy(
DAG.getNode(ISD::ADD, dl, SrcVT, Src,
DAG.getConstant(Offset, dl,
SrcVT)),
- DAG.getConstant(Repeats.BytesLeft, dl,
+ DAG.getConstant(Repeats.BytesLeft(), dl,
SizeVT),
Align, isVolatile, AlwaysInline, false,
DstPtrInfo.getWithOffset(Offset),
diff --git a/lib/Target/X86/X86Subtarget.cpp b/lib/Target/X86/X86Subtarget.cpp
index 4154530d04e72..82ff436f7ebf3 100644
--- a/lib/Target/X86/X86Subtarget.cpp
+++ b/lib/Target/X86/X86Subtarget.cpp
@@ -290,6 +290,9 @@ void X86Subtarget::initializeEnvironment() {
HasMWAITX = false;
HasCLZERO = false;
HasMPX = false;
+ HasSGX = false;
+ HasCLFLUSHOPT = false;
+ HasCLWB = false;
IsBTMemSlow = false;
IsPMULLDSlow = false;
IsSHLDSlow = false;
@@ -326,7 +329,8 @@ X86Subtarget &X86Subtarget::initializeSubtargetDependencies(StringRef CPU,
X86Subtarget::X86Subtarget(const Triple &TT, StringRef CPU, StringRef FS,
const X86TargetMachine &TM,
- unsigned StackAlignOverride)
+ unsigned StackAlignOverride, bool OptForSize,
+ bool OptForMinSize)
: X86GenSubtargetInfo(TT, CPU, FS), X86ProcFamily(Others),
PICStyle(PICStyles::None), TM(TM), TargetTriple(TT),
StackAlignOverride(StackAlignOverride),
@@ -335,8 +339,9 @@ X86Subtarget::X86Subtarget(const Triple &TT, StringRef CPU, StringRef FS,
TargetTriple.getEnvironment() != Triple::CODE16),
In16BitMode(TargetTriple.getArch() == Triple::x86 &&
TargetTriple.getEnvironment() == Triple::CODE16),
- InstrInfo(initializeSubtargetDependencies(CPU, FS)),
- TLInfo(TM, *this), FrameLowering(*this, getStackAlignment()) {
+ InstrInfo(initializeSubtargetDependencies(CPU, FS)), TLInfo(TM, *this),
+ FrameLowering(*this, getStackAlignment()), OptForSize(OptForSize),
+ OptForMinSize(OptForMinSize) {
// Determine the PICStyle based on the target selected.
if (!isPositionIndependent())
setPICStyle(PICStyles::None);
diff --git a/lib/Target/X86/X86Subtarget.h b/lib/Target/X86/X86Subtarget.h
index fd057f36c8900..8568cf04e7d21 100644
--- a/lib/Target/X86/X86Subtarget.h
+++ b/lib/Target/X86/X86Subtarget.h
@@ -328,12 +328,16 @@ private:
X86TargetLowering TLInfo;
X86FrameLowering FrameLowering;
+ bool OptForSize;
+ bool OptForMinSize;
+
public:
/// This constructor initializes the data members to match that
/// of the specified triple.
///
X86Subtarget(const Triple &TT, StringRef CPU, StringRef FS,
- const X86TargetMachine &TM, unsigned StackAlignOverride);
+ const X86TargetMachine &TM, unsigned StackAlignOverride,
+ bool OptForSize, bool OptForMinSize);
/// This object will take onwership of \p GISelAccessor.
void setGISelAccessor(GISelAccessor &GISel) { this->GISel.reset(&GISel); }
@@ -499,6 +503,9 @@ public:
bool isSLM() const { return X86ProcFamily == IntelSLM; }
bool useSoftFloat() const { return UseSoftFloat; }
+ bool getOptForSize() const { return OptForSize; }
+ bool getOptForMinSize() const { return OptForMinSize; }
+
/// Use mfence if we have SSE2 or we're on x86-64 (even if we asked for
/// no-sse2). There isn't any reason to disable it if the target processor
/// supports it.
diff --git a/lib/Target/X86/X86TargetMachine.cpp b/lib/Target/X86/X86TargetMachine.cpp
index 623cf38aa9511..086f55dd60b5f 100644
--- a/lib/Target/X86/X86TargetMachine.cpp
+++ b/lib/Target/X86/X86TargetMachine.cpp
@@ -268,6 +268,12 @@ X86TargetMachine::getSubtargetImpl(const Function &F) const {
FS = Key.substr(CPU.size());
+ bool OptForSize = F.optForSize();
+ bool OptForMinSize = F.optForMinSize();
+
+ Key += std::string(OptForSize ? "+" : "-") + "optforsize";
+ Key += std::string(OptForMinSize ? "+" : "-") + "optforminsize";
+
auto &I = SubtargetMap[Key];
if (!I) {
// This needs to be done before we create a new subtarget since any
@@ -275,7 +281,8 @@ X86TargetMachine::getSubtargetImpl(const Function &F) const {
// function that reside in TargetOptions.
resetTargetOptions(F);
I = llvm::make_unique<X86Subtarget>(TargetTriple, CPU, FS, *this,
- Options.StackAlignmentOverride);
+ Options.StackAlignmentOverride,
+ OptForSize, OptForMinSize);
#ifndef LLVM_BUILD_GLOBAL_ISEL
GISelAccessor *GISel = new GISelAccessor();
#else
@@ -286,7 +293,8 @@ X86TargetMachine::getSubtargetImpl(const Function &F) const {
auto *RBI = new X86RegisterBankInfo(*I->getRegisterInfo());
GISel->RegBankInfo.reset(RBI);
- GISel->InstSelector.reset(createX86InstructionSelector(*this, *I, *RBI));
+ GISel->InstSelector.reset(createX86InstructionSelector(
+ *this, *I, *RBI));
#endif
I->setGISelAccessor(*GISel);
}
generated by cgit v1.3 (git 2.53.0) at 2026-09-08 06:36:45 +0000