aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/IR
diff options
context:
space:
mode:
authorDimitry Andric <dim@FreeBSD.org>2021-02-16 20:13:02 +0000
committerDimitry Andric <dim@FreeBSD.org>2021-02-16 20:13:02 +0000
commitb60736ec1405bb0a8dd40989f67ef4c93da068ab (patch)
tree5c43fbb7c9fc45f0f87e0e6795a86267dbd12f9d /llvm/lib/IR
parentcfca06d7963fa0909f90483b42a6d7d194d01e08 (diff)
downloadsrc-b60736ec1405bb0a8dd40989f67ef4c93da068ab.tar.gz
src-b60736ec1405bb0a8dd40989f67ef4c93da068ab.zip
vendor/llvm-project/llvmorg-12-init-17869-g8e464dd76bef
Diffstat (limited to 'llvm/lib/IR')
-rw-r--r--llvm/lib/IR/AsmWriter.cpp146
-rw-r--r--llvm/lib/IR/Assumptions.cpp36
-rw-r--r--llvm/lib/IR/AttributeImpl.h7
-rw-r--r--llvm/lib/IR/Attributes.cpp241
-rw-r--r--llvm/lib/IR/AutoUpgrade.cpp448
-rw-r--r--llvm/lib/IR/BasicBlock.cpp142
-rw-r--r--llvm/lib/IR/ConstantFold.cpp186
-rw-r--r--llvm/lib/IR/ConstantRange.cpp97
-rw-r--r--llvm/lib/IR/Constants.cpp403
-rw-r--r--llvm/lib/IR/Core.cpp51
-rw-r--r--llvm/lib/IR/DIBuilder.cpp47
-rw-r--r--llvm/lib/IR/DataLayout.cpp414
-rw-r--r--llvm/lib/IR/DebugInfo.cpp35
-rw-r--r--llvm/lib/IR/DebugInfoMetadata.cpp142
-rw-r--r--llvm/lib/IR/DebugLoc.cpp18
-rw-r--r--llvm/lib/IR/DiagnosticInfo.cpp26
-rw-r--r--llvm/lib/IR/Dominators.cpp24
-rw-r--r--llvm/lib/IR/Function.cpp128
-rw-r--r--llvm/lib/IR/Globals.cpp11
-rw-r--r--llvm/lib/IR/IRBuilder.cpp148
-rw-r--r--llvm/lib/IR/IRPrintingPasses.cpp4
-rw-r--r--llvm/lib/IR/Instruction.cpp21
-rw-r--r--llvm/lib/IR/Instructions.cpp338
-rw-r--r--llvm/lib/IR/IntrinsicInst.cpp37
-rw-r--r--llvm/lib/IR/LLVMContext.cpp9
-rw-r--r--llvm/lib/IR/LLVMContextImpl.cpp40
-rw-r--r--llvm/lib/IR/LLVMContextImpl.h225
-rw-r--r--llvm/lib/IR/LLVMRemarkStreamer.cpp17
-rw-r--r--llvm/lib/IR/LegacyPassManager.cpp211
-rw-r--r--llvm/lib/IR/MDBuilder.cpp32
-rw-r--r--llvm/lib/IR/Mangler.cpp32
-rw-r--r--llvm/lib/IR/Metadata.cpp391
-rw-r--r--llvm/lib/IR/Module.cpp9
-rw-r--r--llvm/lib/IR/ModuleSummaryIndex.cpp28
-rw-r--r--llvm/lib/IR/OptBisect.cpp2
-rw-r--r--llvm/lib/IR/Pass.cpp13
-rw-r--r--llvm/lib/IR/PassInstrumentation.cpp19
-rw-r--r--llvm/lib/IR/PassManager.cpp48
-rw-r--r--llvm/lib/IR/PassRegistry.cpp6
-rw-r--r--llvm/lib/IR/PassTimingInfo.cpp60
-rw-r--r--llvm/lib/IR/PrintPasses.cpp88
-rw-r--r--llvm/lib/IR/ProfileSummary.cpp2
-rw-r--r--llvm/lib/IR/PseudoProbe.cpp58
-rw-r--r--llvm/lib/IR/ReplaceConstant.cpp70
-rw-r--r--llvm/lib/IR/SafepointIRVerifier.cpp3
-rw-r--r--llvm/lib/IR/StructuralHash.cpp84
-rw-r--r--llvm/lib/IR/Type.cpp63
-rw-r--r--llvm/lib/IR/Use.cpp29
-rw-r--r--llvm/lib/IR/User.cpp2
-rw-r--r--llvm/lib/IR/Value.cpp86
-rw-r--r--llvm/lib/IR/Verifier.cpp402
51 files changed, 3443 insertions, 1736 deletions
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index fd08310316b3..69abf8769e4b 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -116,6 +116,15 @@ struct OrderMap {
} // end anonymous namespace
+/// Look for a value that might be wrapped as metadata, e.g. a value in a
+/// metadata operand. Returns the input value as-is if it is not wrapped.
+static const Value *skipMetadataWrapper(const Value *V) {
+ if (const auto *MAV = dyn_cast<MetadataAsValue>(V))
+ if (const auto *VAM = dyn_cast<ValueAsMetadata>(MAV->getMetadata()))
+ return VAM->getValue();
+ return V;
+}
+
static void orderValue(const Value *V, OrderMap &OM) {
if (OM.lookup(V).first)
return;
@@ -132,8 +141,6 @@ static void orderValue(const Value *V, OrderMap &OM) {
}
static OrderMap orderModule(const Module *M) {
- // This needs to match the order used by ValueEnumerator::ValueEnumerator()
- // and ValueEnumerator::incorporateFunction().
OrderMap OM;
for (const GlobalVariable &G : M->globals()) {
@@ -167,10 +174,12 @@ static OrderMap orderModule(const Module *M) {
for (const BasicBlock &BB : F) {
orderValue(&BB, OM);
for (const Instruction &I : BB) {
- for (const Value *Op : I.operands())
+ for (const Value *Op : I.operands()) {
+ Op = skipMetadataWrapper(Op);
if ((isa<Constant>(*Op) && !isa<GlobalValue>(*Op)) ||
isa<InlineAsm>(*Op))
orderValue(Op, OM);
+ }
orderValue(&I, OM);
}
}
@@ -284,9 +293,11 @@ static UseListOrderStack predictUseListOrder(const Module *M) {
predictValueUseListOrder(&A, &F, OM, Stack);
for (const BasicBlock &BB : F)
for (const Instruction &I : BB)
- for (const Value *Op : I.operands())
+ for (const Value *Op : I.operands()) {
+ Op = skipMetadataWrapper(Op);
if (isa<Constant>(*Op) || isa<InlineAsm>(*Op)) // Visit GlobalValues.
predictValueUseListOrder(Op, &F, OM, Stack);
+ }
for (const BasicBlock &BB : F)
for (const Instruction &I : BB)
predictValueUseListOrder(&I, &F, OM, Stack);
@@ -388,6 +399,7 @@ static void PrintCallingConv(unsigned cc, raw_ostream &Out) {
case CallingConv::AMDGPU_PS: Out << "amdgpu_ps"; break;
case CallingConv::AMDGPU_CS: Out << "amdgpu_cs"; break;
case CallingConv::AMDGPU_KERNEL: Out << "amdgpu_kernel"; break;
+ case CallingConv::AMDGPU_Gfx: Out << "amdgpu_gfx"; break;
}
}
@@ -597,6 +609,7 @@ void TypePrinting::print(Type *Ty, raw_ostream &OS) {
case Type::LabelTyID: OS << "label"; return;
case Type::MetadataTyID: OS << "metadata"; return;
case Type::X86_MMXTyID: OS << "x86_mmx"; return;
+ case Type::X86_AMXTyID: OS << "x86_amx"; return;
case Type::TokenTyID: OS << "token"; return;
case Type::IntegerTyID:
OS << 'i' << cast<IntegerType>(Ty)->getBitWidth();
@@ -656,9 +669,9 @@ void TypePrinting::print(Type *Ty, raw_ostream &OS) {
VectorType *PTy = cast<VectorType>(Ty);
ElementCount EC = PTy->getElementCount();
OS << "<";
- if (EC.Scalable)
+ if (EC.isScalable())
OS << "vscale x ";
- OS << EC.Min << " x ";
+ OS << EC.getKnownMinValue() << " x ";
print(PTy->getElementType(), OS);
OS << '>';
return;
@@ -1355,9 +1368,8 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV,
// "Inf" or NaN, that atof will accept, but the lexer will not. Check
// that the string matches the "[-+]?[0-9]" regex.
//
- assert(((StrVal[0] >= '0' && StrVal[0] <= '9') ||
- ((StrVal[0] == '-' || StrVal[0] == '+') &&
- (StrVal[1] >= '0' && StrVal[1] <= '9'))) &&
+ assert((isDigit(StrVal[0]) || ((StrVal[0] == '-' || StrVal[0] == '+') &&
+ isDigit(StrVal[1]))) &&
"[-+]?[0-9] regex does not match!");
// Reparse stringized version!
if (APFloat(APFloat::IEEEdouble(), StrVal).convertToDouble() == Val) {
@@ -1373,9 +1385,19 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV,
"assuming that double is 64 bits!");
APFloat apf = APF;
// Floats are represented in ASCII IR as double, convert.
- if (!isDouble)
+ // FIXME: We should allow 32-bit hex float and remove this.
+ if (!isDouble) {
+ // A signaling NaN is quieted on conversion, so we need to recreate the
+ // expected value after convert (quiet bit of the payload is clear).
+ bool IsSNAN = apf.isSignaling();
apf.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
- &ignored);
+ &ignored);
+ if (IsSNAN) {
+ APInt Payload = apf.bitcastToAPInt();
+ apf = APFloat::getSNaN(APFloat::IEEEdouble(), apf.isNegative(),
+ &Payload);
+ }
+ }
Out << format_hex(apf.bitcastToAPInt().getZExtValue(), 0, /*Upper=*/true);
return;
}
@@ -1433,6 +1455,13 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV,
return;
}
+ if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(CV)) {
+ Out << "dso_local_equivalent ";
+ WriteAsOperandInternal(Out, Equiv->getGlobalValue(), &TypePrinter, Machine,
+ Context);
+ return;
+ }
+
if (const ConstantArray *CA = dyn_cast<ConstantArray>(CV)) {
Type *ETy = CA->getType()->getElementType();
Out << '[';
@@ -1511,7 +1540,7 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV,
}
if (isa<ConstantVector>(CV) || isa<ConstantDataVector>(CV)) {
- auto *CVVTy = cast<VectorType>(CV->getType());
+ auto *CVVTy = cast<FixedVectorType>(CV->getType());
Type *ETy = CVVTy->getElementType();
Out << '<';
TypePrinter.print(ETy, Out);
@@ -1539,6 +1568,11 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV,
return;
}
+ if (isa<PoisonValue>(CV)) {
+ Out << "poison";
+ return;
+ }
+
if (isa<UndefValue>(CV)) {
Out << "undef";
return;
@@ -1889,6 +1923,57 @@ static void writeDISubrange(raw_ostream &Out, const DISubrange *N,
Out << ")";
}
+static void writeDIGenericSubrange(raw_ostream &Out, const DIGenericSubrange *N,
+ TypePrinting *TypePrinter,
+ SlotTracker *Machine,
+ const Module *Context) {
+ Out << "!DIGenericSubrange(";
+ MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
+
+ auto IsConstant = [&](Metadata *Bound) -> bool {
+ if (auto *BE = dyn_cast_or_null<DIExpression>(Bound)) {
+ return BE->isSignedConstant();
+ }
+ return false;
+ };
+
+ auto GetConstant = [&](Metadata *Bound) -> int64_t {
+ assert(IsConstant(Bound) && "Expected constant");
+ auto *BE = dyn_cast_or_null<DIExpression>(Bound);
+ return static_cast<int64_t>(BE->getElement(1));
+ };
+
+ auto *Count = N->getRawCountNode();
+ if (IsConstant(Count))
+ Printer.printInt("count", GetConstant(Count),
+ /* ShouldSkipZero */ false);
+ else
+ Printer.printMetadata("count", Count, /*ShouldSkipNull */ true);
+
+ auto *LBound = N->getRawLowerBound();
+ if (IsConstant(LBound))
+ Printer.printInt("lowerBound", GetConstant(LBound),
+ /* ShouldSkipZero */ false);
+ else
+ Printer.printMetadata("lowerBound", LBound, /*ShouldSkipNull */ true);
+
+ auto *UBound = N->getRawUpperBound();
+ if (IsConstant(UBound))
+ Printer.printInt("upperBound", GetConstant(UBound),
+ /* ShouldSkipZero */ false);
+ else
+ Printer.printMetadata("upperBound", UBound, /*ShouldSkipNull */ true);
+
+ auto *Stride = N->getRawStride();
+ if (IsConstant(Stride))
+ Printer.printInt("stride", GetConstant(Stride),
+ /* ShouldSkipZero */ false);
+ else
+ Printer.printMetadata("stride", Stride, /*ShouldSkipNull */ true);
+
+ Out << ")";
+}
+
static void writeDIEnumerator(raw_ostream &Out, const DIEnumerator *N,
TypePrinting *, SlotTracker *, const Module *) {
Out << "!DIEnumerator(";
@@ -1916,6 +2001,23 @@ static void writeDIBasicType(raw_ostream &Out, const DIBasicType *N,
Out << ")";
}
+static void writeDIStringType(raw_ostream &Out, const DIStringType *N,
+ TypePrinting *TypePrinter, SlotTracker *Machine,
+ const Module *Context) {
+ Out << "!DIStringType(";
+ MDFieldPrinter Printer(Out, TypePrinter, Machine, Context);
+ if (N->getTag() != dwarf::DW_TAG_string_type)
+ Printer.printTag(N);
+ Printer.printString("name", N->getName());
+ Printer.printMetadata("stringLength", N->getRawStringLength());
+ Printer.printMetadata("stringLengthExpression", N->getRawStringLengthExp());
+ Printer.printInt("size", N->getSizeInBits());
+ Printer.printInt("align", N->getAlignInBits());
+ Printer.printDwarfEnum("encoding", N->getEncoding(),
+ dwarf::AttributeEncodingString);
+ Out << ")";
+}
+
static void writeDIDerivedType(raw_ostream &Out, const DIDerivedType *N,
TypePrinting *TypePrinter, SlotTracker *Machine,
const Module *Context) {
@@ -1962,6 +2064,13 @@ static void writeDICompositeType(raw_ostream &Out, const DICompositeType *N,
Printer.printString("identifier", N->getIdentifier());
Printer.printMetadata("discriminator", N->getRawDiscriminator());
Printer.printMetadata("dataLocation", N->getRawDataLocation());
+ Printer.printMetadata("associated", N->getRawAssociated());
+ Printer.printMetadata("allocated", N->getRawAllocated());
+ if (auto *RankConst = N->getRankConst())
+ Printer.printInt("rank", RankConst->getSExtValue(),
+ /* ShouldSkipZero */ false);
+ else
+ Printer.printMetadata("rank", N->getRawRank(), /*ShouldSkipNull */ true);
Out << ")";
}
@@ -2136,6 +2245,7 @@ static void writeDIModule(raw_ostream &Out, const DIModule *N,
Printer.printString("apinotes", N->getAPINotesFile());
Printer.printMetadata("file", N->getRawFile());
Printer.printInt("line", N->getLineNo());
+ Printer.printBool("isDecl", N->getIsDecl(), /* Default */ false);
Out << ")";
}
@@ -3087,7 +3197,7 @@ void AssemblyWriter::printFunctionSummary(const FunctionSummary *FS) {
printTypeIdInfo(*TIdInfo);
auto PrintRange = [&](const ConstantRange &Range) {
- Out << "[" << Range.getLower() << ", " << Range.getSignedMax() << "]";
+ Out << "[" << Range.getSignedMin() << ", " << Range.getSignedMax() << "]";
};
if (!FS->paramAccesses().empty()) {
@@ -3103,7 +3213,7 @@ void AssemblyWriter::printFunctionSummary(const FunctionSummary *FS) {
FieldSeparator IFS;
for (auto &Call : PS.Calls) {
Out << IFS;
- Out << "(callee: ^" << Machine.getGUIDSlot(Call.Callee);
+ Out << "(callee: ^" << Machine.getGUIDSlot(Call.Callee.getGUID());
Out << ", param: " << Call.ParamNo;
Out << ", offset: ";
PrintRange(Call.Offsets);
@@ -4269,11 +4379,17 @@ void AssemblyWriter::writeAttribute(const Attribute &Attr, bool InAttrGroup) {
}
assert((Attr.hasAttribute(Attribute::ByVal) ||
+ Attr.hasAttribute(Attribute::StructRet) ||
+ Attr.hasAttribute(Attribute::ByRef) ||
Attr.hasAttribute(Attribute::Preallocated)) &&
"unexpected type attr");
if (Attr.hasAttribute(Attribute::ByVal)) {
Out << "byval";
+ } else if (Attr.hasAttribute(Attribute::StructRet)) {
+ Out << "sret";
+ } else if (Attr.hasAttribute(Attribute::ByRef)) {
+ Out << "byref";
} else {
Out << "preallocated";
}
@@ -4366,7 +4482,7 @@ void Function::print(raw_ostream &ROS, AssemblyAnnotationWriter *AAW,
void BasicBlock::print(raw_ostream &ROS, AssemblyAnnotationWriter *AAW,
bool ShouldPreserveUseListOrder,
bool IsForDebug) const {
- SlotTracker SlotTable(this->getModule());
+ SlotTracker SlotTable(this->getParent());
formatted_raw_ostream OS(ROS);
AssemblyWriter W(OS, SlotTable, this->getModule(), AAW,
IsForDebug,
diff --git a/llvm/lib/IR/Assumptions.cpp b/llvm/lib/IR/Assumptions.cpp
new file mode 100644
index 000000000000..1bd8b7f51e67
--- /dev/null
+++ b/llvm/lib/IR/Assumptions.cpp
@@ -0,0 +1,36 @@
+//===- Assumptions.cpp ------ Collection of helpers for assumptions -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/IR/Assumptions.h"
+#include "llvm/IR/Attributes.h"
+#include "llvm/IR/Function.h"
+
+using namespace llvm;
+
+bool llvm::hasAssumption(Function &F,
+ const KnownAssumptionString &AssumptionStr) {
+ const Attribute &A = F.getFnAttribute(AssumptionAttrKey);
+ if (!A.isValid())
+ return false;
+ assert(A.isStringAttribute() && "Expected a string attribute!");
+
+ SmallVector<StringRef, 8> Strings;
+ A.getValueAsString().split(Strings, ",");
+
+ return llvm::any_of(Strings, [=](StringRef Assumption) {
+ return Assumption == AssumptionStr;
+ });
+}
+
+StringSet<> llvm::KnownAssumptionStrings({
+ "omp_no_openmp", // OpenMP 5.1
+ "omp_no_openmp_routines", // OpenMP 5.1
+ "omp_no_parallelism", // OpenMP 5.1
+});
diff --git a/llvm/lib/IR/AttributeImpl.h b/llvm/lib/IR/AttributeImpl.h
index 5c334348cde3..c69fe3fe0827 100644
--- a/llvm/lib/IR/AttributeImpl.h
+++ b/llvm/lib/IR/AttributeImpl.h
@@ -121,7 +121,10 @@ protected:
public:
EnumAttributeImpl(Attribute::AttrKind Kind)
- : AttributeImpl(EnumAttrEntry), Kind(Kind) {}
+ : AttributeImpl(EnumAttrEntry), Kind(Kind) {
+ assert(Kind != Attribute::AttrKind::None &&
+ "Can't create a None attribute!");
+ }
Attribute::AttrKind getEnumKind() const { return Kind; }
};
@@ -251,6 +254,8 @@ public:
std::pair<unsigned, Optional<unsigned>> getAllocSizeArgs() const;
std::string getAsString(bool InAttrGrp) const;
Type *getByValType() const;
+ Type *getStructRetType() const;
+ Type *getByRefType() const;
Type *getPreallocatedType() const;
using iterator = const Attribute *;
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index f67d96a854f4..c4629decc6d9 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -172,6 +172,14 @@ Attribute Attribute::getWithByValType(LLVMContext &Context, Type *Ty) {
return get(Context, ByVal, Ty);
}
+Attribute Attribute::getWithStructRetType(LLVMContext &Context, Type *Ty) {
+ return get(Context, StructRet, Ty);
+}
+
+Attribute Attribute::getWithByRefType(LLVMContext &Context, Type *Ty) {
+ return get(Context, ByRef, Ty);
+}
+
Attribute Attribute::getWithPreallocatedType(LLVMContext &Context, Type *Ty) {
return get(Context, Preallocated, Ty);
}
@@ -363,6 +371,8 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
return "noalias";
if (hasAttribute(Attribute::NoBuiltin))
return "nobuiltin";
+ if (hasAttribute(Attribute::NoCallback))
+ return "nocallback";
if (hasAttribute(Attribute::NoCapture))
return "nocapture";
if (hasAttribute(Attribute::NoDuplicate))
@@ -393,6 +403,8 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
return "nocf_check";
if (hasAttribute(Attribute::NoRecurse))
return "norecurse";
+ if (hasAttribute(Attribute::NoProfile))
+ return "noprofile";
if (hasAttribute(Attribute::NoUnwind))
return "nounwind";
if (hasAttribute(Attribute::OptForFuzzing))
@@ -429,8 +441,6 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
return "shadowcallstack";
if (hasAttribute(Attribute::StrictFP))
return "strictfp";
- if (hasAttribute(Attribute::StructRet))
- return "sret";
if (hasAttribute(Attribute::SanitizeThread))
return "sanitize_thread";
if (hasAttribute(Attribute::SanitizeMemory))
@@ -441,14 +451,19 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
return "zeroext";
if (hasAttribute(Attribute::Cold))
return "cold";
+ if (hasAttribute(Attribute::Hot))
+ return "hot";
if (hasAttribute(Attribute::ImmArg))
return "immarg";
if (hasAttribute(Attribute::NoUndef))
return "noundef";
+ if (hasAttribute(Attribute::MustProgress))
+ return "mustprogress";
- if (hasAttribute(Attribute::ByVal)) {
+ const bool IsByVal = hasAttribute(Attribute::ByVal);
+ if (IsByVal || hasAttribute(Attribute::StructRet)) {
std::string Result;
- Result += "byval";
+ Result += IsByVal ? "byval" : "sret";
if (Type *Ty = getValueAsType()) {
raw_string_ostream OS(Result);
Result += '(';
@@ -459,9 +474,9 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
return Result;
}
- if (hasAttribute(Attribute::Preallocated)) {
- std::string Result;
- Result += "preallocated";
+ const bool IsByRef = hasAttribute(Attribute::ByRef);
+ if (IsByRef || hasAttribute(Attribute::Preallocated)) {
+ std::string Result = IsByRef ? "byref" : "preallocated";
raw_string_ostream OS(Result);
Result += '(';
getValueAsType()->print(OS, false, true);
@@ -742,10 +757,18 @@ uint64_t AttributeSet::getDereferenceableOrNullBytes() const {
return SetNode ? SetNode->getDereferenceableOrNullBytes() : 0;
}
+Type *AttributeSet::getByRefType() const {
+ return SetNode ? SetNode->getByRefType() : nullptr;
+}
+
Type *AttributeSet::getByValType() const {
return SetNode ? SetNode->getByValType() : nullptr;
}
+Type *AttributeSet::getStructRetType() const {
+ return SetNode ? SetNode->getStructRetType() : nullptr;
+}
+
Type *AttributeSet::getPreallocatedType() const {
return SetNode ? SetNode->getPreallocatedType() : nullptr;
}
@@ -842,6 +865,12 @@ AttributeSetNode *AttributeSetNode::get(LLVMContext &C, const AttrBuilder &B) {
case Attribute::ByVal:
Attr = Attribute::getWithByValType(C, B.getByValType());
break;
+ case Attribute::StructRet:
+ Attr = Attribute::getWithStructRetType(C, B.getStructRetType());
+ break;
+ case Attribute::ByRef:
+ Attr = Attribute::getWithByRefType(C, B.getByRefType());
+ break;
case Attribute::Preallocated:
Attr = Attribute::getWithPreallocatedType(C, B.getPreallocatedType());
break;
@@ -925,14 +954,25 @@ MaybeAlign AttributeSetNode::getStackAlignment() const {
Type *AttributeSetNode::getByValType() const {
if (auto A = findEnumAttribute(Attribute::ByVal))
return A->getValueAsType();
- return 0;
+ return nullptr;
+}
+
+Type *AttributeSetNode::getStructRetType() const {
+ if (auto A = findEnumAttribute(Attribute::StructRet))
+ return A->getValueAsType();
+ return nullptr;
+}
+
+Type *AttributeSetNode::getByRefType() const {
+ if (auto A = findEnumAttribute(Attribute::ByRef))
+ return A->getValueAsType();
+ return nullptr;
}
Type *AttributeSetNode::getPreallocatedType() const {
- for (const auto &I : *this)
- if (I.hasAttribute(Attribute::Preallocated))
- return I.getValueAsType();
- return 0;
+ if (auto A = findEnumAttribute(Attribute::Preallocated))
+ return A->getValueAsType();
+ return nullptr;
}
uint64_t AttributeSetNode::getDereferenceableBytes() const {
@@ -970,7 +1010,7 @@ std::string AttributeSetNode::getAsString(bool InAttrGrp) const {
/// Map from AttributeList index to the internal array index. Adding one happens
/// to work, because -1 wraps around to 0.
-static constexpr unsigned attrIdxToArrayIdx(unsigned Index) {
+static unsigned attrIdxToArrayIdx(unsigned Index) {
return Index + 1;
}
@@ -983,9 +1023,7 @@ AttributeListImpl::AttributeListImpl(ArrayRef<AttributeSet> Sets)
// Initialize AvailableFunctionAttrs and AvailableSomewhereAttrs
// summary bitsets.
- static_assert(attrIdxToArrayIdx(AttributeList::FunctionIndex) == 0U,
- "function should be stored in slot 0");
- for (const auto &I : Sets[0])
+ for (const auto &I : Sets[attrIdxToArrayIdx(AttributeList::FunctionIndex)])
if (!I.isStringAttribute())
AvailableFunctionAttrs.addAttribute(I.getKindAsEnum());
@@ -1073,10 +1111,10 @@ AttributeList::get(LLVMContext &C,
return LHS.first < RHS.first;
}) &&
"Misordered Attributes list!");
- assert(llvm::none_of(Attrs,
- [](const std::pair<unsigned, Attribute> &Pair) {
- return Pair.second.hasAttribute(Attribute::None);
- }) &&
+ assert(llvm::all_of(Attrs,
+ [](const std::pair<unsigned, Attribute> &Pair) {
+ return Pair.second.isValid();
+ }) &&
"Pointless attribute!");
// Create a vector if (unsigned, AttributeSetNode*) pairs from the attributes
@@ -1164,7 +1202,7 @@ AttributeList AttributeList::get(LLVMContext &C, AttributeSet FnAttrs,
if (NumSets > 2) {
// Drop the empty argument attribute sets at the end.
ArgAttrs = ArgAttrs.take_front(NumSets - 2);
- AttrSets.insert(AttrSets.end(), ArgAttrs.begin(), ArgAttrs.end());
+ llvm::append_range(AttrSets, ArgAttrs);
}
return getImpl(C, AttrSets);
@@ -1236,9 +1274,11 @@ AttributeList AttributeList::get(LLVMContext &C,
AttributeList AttributeList::addAttribute(LLVMContext &C, unsigned Index,
Attribute::AttrKind Kind) const {
if (hasAttribute(Index, Kind)) return *this;
- AttrBuilder B;
- B.addAttribute(Kind);
- return addAttributes(C, Index, B);
+ AttributeSet Attrs = getAttributes(Index);
+ // TODO: Insert at correct position and avoid sort.
+ SmallVector<Attribute, 8> NewAttrs(Attrs.begin(), Attrs.end());
+ NewAttrs.push_back(Attribute::get(C, Kind));
+ return setAttributes(C, Index, AttributeSet::get(C, NewAttrs));
}
AttributeList AttributeList::addAttribute(LLVMContext &C, unsigned Index,
@@ -1256,6 +1296,16 @@ AttributeList AttributeList::addAttribute(LLVMContext &C, unsigned Index,
return addAttributes(C, Index, B);
}
+AttributeList AttributeList::setAttributes(LLVMContext &C, unsigned Index,
+ AttributeSet Attrs) const {
+ Index = attrIdxToArrayIdx(Index);
+ SmallVector<AttributeSet, 4> AttrSets(this->begin(), this->end());
+ if (Index >= AttrSets.size())
+ AttrSets.resize(Index + 1);
+ AttrSets[Index] = Attrs;
+ return AttributeList::getImpl(C, AttrSets);
+}
+
AttributeList AttributeList::addAttributes(LLVMContext &C, unsigned Index,
const AttrBuilder &B) const {
if (!B.hasAttributes())
@@ -1273,16 +1323,9 @@ AttributeList AttributeList::addAttributes(LLVMContext &C, unsigned Index,
"Attempt to change alignment!");
#endif
- Index = attrIdxToArrayIdx(Index);
- SmallVector<AttributeSet, 4> AttrSets(this->begin(), this->end());
- if (Index >= AttrSets.size())
- AttrSets.resize(Index + 1);
-
- AttrBuilder Merged(AttrSets[Index]);
+ AttrBuilder Merged(getAttributes(Index));
Merged.merge(B);
- AttrSets[Index] = AttributeSet::get(C, Merged);
-
- return getImpl(C, AttrSets);
+ return setAttributes(C, Index, AttributeSet::get(C, Merged));
}
AttributeList AttributeList::addParamAttribute(LLVMContext &C,
@@ -1452,6 +1495,14 @@ Type *AttributeList::getParamByValType(unsigned Index) const {
return getAttributes(Index+FirstArgIndex).getByValType();
}
+Type *AttributeList::getParamStructRetType(unsigned Index) const {
+ return getAttributes(Index + FirstArgIndex).getStructRetType();
+}
+
+Type *AttributeList::getParamByRefType(unsigned Index) const {
+ return getAttributes(Index + FirstArgIndex).getByRefType();
+}
+
Type *AttributeList::getParamPreallocatedType(unsigned Index) const {
return getAttributes(Index + FirstArgIndex).getPreallocatedType();
}
@@ -1537,17 +1588,11 @@ void AttrBuilder::clear() {
DerefBytes = DerefOrNullBytes = 0;
AllocSizeArgs = 0;
ByValType = nullptr;
+ StructRetType = nullptr;
+ ByRefType = nullptr;
PreallocatedType = nullptr;
}
-AttrBuilder &AttrBuilder::addAttribute(Attribute::AttrKind Val) {
- assert((unsigned)Val < Attribute::EndAttrKinds && "Attribute out of range!");
- assert(!Attribute::doesAttrKindHaveArgument(Val) &&
- "Adding integer attribute without adding a value!");
- Attrs[Val] = true;
- return *this;
-}
-
AttrBuilder &AttrBuilder::addAttribute(Attribute Attr) {
if (Attr.isStringAttribute()) {
addAttribute(Attr.getKindAsString(), Attr.getValueAsString());
@@ -1563,6 +1608,10 @@ AttrBuilder &AttrBuilder::addAttribute(Attribute Attr) {
StackAlignment = Attr.getStackAlignment();
else if (Kind == Attribute::ByVal)
ByValType = Attr.getValueAsType();
+ else if (Kind == Attribute::StructRet)
+ StructRetType = Attr.getValueAsType();
+ else if (Kind == Attribute::ByRef)
+ ByRefType = Attr.getValueAsType();
else if (Kind == Attribute::Preallocated)
PreallocatedType = Attr.getValueAsType();
else if (Kind == Attribute::Dereferenceable)
@@ -1589,6 +1638,10 @@ AttrBuilder &AttrBuilder::removeAttribute(Attribute::AttrKind Val) {
StackAlignment.reset();
else if (Val == Attribute::ByVal)
ByValType = nullptr;
+ else if (Val == Attribute::StructRet)
+ StructRetType = nullptr;
+ else if (Val == Attribute::ByRef)
+ ByRefType = nullptr;
else if (Val == Attribute::Preallocated)
PreallocatedType = nullptr;
else if (Val == Attribute::Dereferenceable)
@@ -1679,6 +1732,18 @@ AttrBuilder &AttrBuilder::addByValAttr(Type *Ty) {
return *this;
}
+AttrBuilder &AttrBuilder::addStructRetAttr(Type *Ty) {
+ Attrs[Attribute::StructRet] = true;
+ StructRetType = Ty;
+ return *this;
+}
+
+AttrBuilder &AttrBuilder::addByRefAttr(Type *Ty) {
+ Attrs[Attribute::ByRef] = true;
+ ByRefType = Ty;
+ return *this;
+}
+
AttrBuilder &AttrBuilder::addPreallocatedAttr(Type *Ty) {
Attrs[Attribute::Preallocated] = true;
PreallocatedType = Ty;
@@ -1705,6 +1770,12 @@ AttrBuilder &AttrBuilder::merge(const AttrBuilder &B) {
if (!ByValType)
ByValType = B.ByValType;
+ if (!StructRetType)
+ StructRetType = B.StructRetType;
+
+ if (!ByRefType)
+ ByRefType = B.ByRefType;
+
if (!PreallocatedType)
PreallocatedType = B.PreallocatedType;
@@ -1736,6 +1807,12 @@ AttrBuilder &AttrBuilder::remove(const AttrBuilder &B) {
if (B.ByValType)
ByValType = nullptr;
+ if (B.StructRetType)
+ StructRetType = nullptr;
+
+ if (B.ByRefType)
+ ByRefType = nullptr;
+
if (B.PreallocatedType)
PreallocatedType = nullptr;
@@ -1788,7 +1865,7 @@ bool AttrBuilder::hasAlignmentAttr() const {
return Alignment != 0;
}
-bool AttrBuilder::operator==(const AttrBuilder &B) {
+bool AttrBuilder::operator==(const AttrBuilder &B) const {
if (Attrs != B.Attrs)
return false;
@@ -1799,6 +1876,7 @@ bool AttrBuilder::operator==(const AttrBuilder &B) {
return Alignment == B.Alignment && StackAlignment == B.StackAlignment &&
DerefBytes == B.DerefBytes && ByValType == B.ByValType &&
+ StructRetType == B.StructRetType && ByRefType == B.ByRefType &&
PreallocatedType == B.PreallocatedType;
}
@@ -1821,14 +1899,20 @@ AttrBuilder AttributeFuncs::typeIncompatible(Type *Ty) {
.addAttribute(Attribute::NoAlias)
.addAttribute(Attribute::NoCapture)
.addAttribute(Attribute::NonNull)
+ .addAlignmentAttr(1) // the int here is ignored
.addDereferenceableAttr(1) // the int here is ignored
.addDereferenceableOrNullAttr(1) // the int here is ignored
.addAttribute(Attribute::ReadNone)
.addAttribute(Attribute::ReadOnly)
- .addAttribute(Attribute::StructRet)
.addAttribute(Attribute::InAlloca)
.addPreallocatedAttr(Ty)
- .addByValAttr(Ty);
+ .addByValAttr(Ty)
+ .addStructRetAttr(Ty)
+ .addByRefAttr(Ty);
+
+ // Some attributes can apply to all "values" but there are no `void` values.
+ if (Ty->isVoidTy())
+ Incompatible.addAttribute(Attribute::NoUndef);
return Incompatible;
}
@@ -1866,6 +1950,16 @@ static void setOR(Function &Caller, const Function &Callee) {
/// If the inlined function had a higher stack protection level than the
/// calling function, then bump up the caller's stack protection level.
static void adjustCallerSSPLevel(Function &Caller, const Function &Callee) {
+#ifndef NDEBUG
+ if (!Callee.hasFnAttribute(Attribute::AlwaysInline)) {
+ assert(!(!Callee.hasStackProtectorFnAttr() &&
+ Caller.hasStackProtectorFnAttr()) &&
+ "stack protected caller but callee requested no stack protector");
+ assert(!(!Caller.hasStackProtectorFnAttr() &&
+ Callee.hasStackProtectorFnAttr()) &&
+ "stack protected callee but caller requested no stack protector");
+ }
+#endif
// If upgrading the SSP attribute, clear out the old SSP Attributes first.
// Having multiple SSP attributes doesn't actually hurt, but it adds useless
// clutter to the IR.
@@ -1901,21 +1995,19 @@ static void adjustCallerStackProbes(Function &Caller, const Function &Callee) {
/// that is no larger.
static void
adjustCallerStackProbeSize(Function &Caller, const Function &Callee) {
- if (Callee.hasFnAttribute("stack-probe-size")) {
- uint64_t CalleeStackProbeSize;
- Callee.getFnAttribute("stack-probe-size")
- .getValueAsString()
- .getAsInteger(0, CalleeStackProbeSize);
- if (Caller.hasFnAttribute("stack-probe-size")) {
- uint64_t CallerStackProbeSize;
- Caller.getFnAttribute("stack-probe-size")
- .getValueAsString()
- .getAsInteger(0, CallerStackProbeSize);
+ Attribute CalleeAttr = Callee.getFnAttribute("stack-probe-size");
+ if (CalleeAttr.isValid()) {
+ Attribute CallerAttr = Caller.getFnAttribute("stack-probe-size");
+ if (CallerAttr.isValid()) {
+ uint64_t CallerStackProbeSize, CalleeStackProbeSize;
+ CallerAttr.getValueAsString().getAsInteger(0, CallerStackProbeSize);
+ CalleeAttr.getValueAsString().getAsInteger(0, CalleeStackProbeSize);
+
if (CallerStackProbeSize > CalleeStackProbeSize) {
- Caller.addFnAttr(Callee.getFnAttribute("stack-probe-size"));
+ Caller.addFnAttr(CalleeAttr);
}
} else {
- Caller.addFnAttr(Callee.getFnAttribute("stack-probe-size"));
+ Caller.addFnAttr(CalleeAttr);
}
}
}
@@ -1931,18 +2023,15 @@ adjustCallerStackProbeSize(Function &Caller, const Function &Callee) {
/// handled as part of inline cost analysis.
static void
adjustMinLegalVectorWidth(Function &Caller, const Function &Callee) {
- if (Caller.hasFnAttribute("min-legal-vector-width")) {
- if (Callee.hasFnAttribute("min-legal-vector-width")) {
- uint64_t CallerVectorWidth;
- Caller.getFnAttribute("min-legal-vector-width")
- .getValueAsString()
- .getAsInteger(0, CallerVectorWidth);
- uint64_t CalleeVectorWidth;
- Callee.getFnAttribute("min-legal-vector-width")
- .getValueAsString()
- .getAsInteger(0, CalleeVectorWidth);
+ Attribute CallerAttr = Caller.getFnAttribute("min-legal-vector-width");
+ if (CallerAttr.isValid()) {
+ Attribute CalleeAttr = Callee.getFnAttribute("min-legal-vector-width");
+ if (CalleeAttr.isValid()) {
+ uint64_t CallerVectorWidth, CalleeVectorWidth;
+ CallerAttr.getValueAsString().getAsInteger(0, CallerVectorWidth);
+ CalleeAttr.getValueAsString().getAsInteger(0, CalleeVectorWidth);
if (CallerVectorWidth < CalleeVectorWidth)
- Caller.addFnAttr(Callee.getFnAttribute("min-legal-vector-width"));
+ Caller.addFnAttr(CalleeAttr);
} else {
// If the callee doesn't have the attribute then we don't know anything
// and must drop the attribute from the caller.
@@ -2009,7 +2098,25 @@ bool AttributeFuncs::areInlineCompatible(const Function &Caller,
return hasCompatibleFnAttrs(Caller, Callee);
}
+bool AttributeFuncs::areOutlineCompatible(const Function &A,
+ const Function &B) {
+ return hasCompatibleFnAttrs(A, B);
+}
+
void AttributeFuncs::mergeAttributesForInlining(Function &Caller,
const Function &Callee) {
mergeFnAttrs(Caller, Callee);
}
+
+void AttributeFuncs::mergeAttributesForOutlining(Function &Base,
+ const Function &ToMerge) {
+
+ // We merge functions so that they meet the most general case.
+ // For example, if the NoNansFPMathAttr is set in one function, but not in
+ // the other, in the merged function we can say that the NoNansFPMathAttr
+ // is not set.
+ // However if we have the SpeculativeLoadHardeningAttr set true in one
+ // function, but not the other, we make sure that the function retains
+ // that aspect in the merged function.
+ mergeFnAttrs(Base, ToMerge);
+}
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 1e8fdb506619..23e7af6287b6 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -23,6 +23,7 @@
#include "llvm/IR/Instruction.h"
#include "llvm/IR/InstVisitor.h"
#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/IntrinsicsARM.h"
#include "llvm/IR/IntrinsicsX86.h"
@@ -68,6 +69,19 @@ static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
return true;
}
+// Upgrade the declaration of fp compare intrinsics that change return type
+// from scalar to vXi1 mask.
+static bool UpgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
+ Function *&NewFn) {
+ // Check if the return type is a vector.
+ if (F->getReturnType()->isVectorTy())
+ return false;
+
+ rename(F);
+ NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
+ return true;
+}
+
static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
// All of the intrinsics matches below should be marked with which llvm
// version started autoupgrading them. At some point in the future we would
@@ -241,7 +255,7 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
Name.startswith("avx512.mask.cmp.d") || // Added in 5.0
Name.startswith("avx512.mask.cmp.q") || // Added in 5.0
Name.startswith("avx512.mask.cmp.w") || // Added in 5.0
- Name.startswith("avx512.mask.cmp.p") || // Added in 7.0
+ Name.startswith("avx512.cmp.p") || // Added in 12.0
Name.startswith("avx512.mask.ucmp.") || // Added in 5.0
Name.startswith("avx512.cvtb2mask.") || // Added in 7.0
Name.startswith("avx512.cvtw2mask.") || // Added in 7.0
@@ -456,6 +470,24 @@ static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name,
if (Name == "avx2.mpsadbw") // Added in 3.6
return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,
NewFn);
+ if (Name == "avx512.mask.cmp.pd.128") // Added in 7.0
+ return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_128,
+ NewFn);
+ if (Name == "avx512.mask.cmp.pd.256") // Added in 7.0
+ return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_256,
+ NewFn);
+ if (Name == "avx512.mask.cmp.pd.512") // Added in 7.0
+ return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_512,
+ NewFn);
+ if (Name == "avx512.mask.cmp.ps.128") // Added in 7.0
+ return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_128,
+ NewFn);
+ if (Name == "avx512.mask.cmp.ps.256") // Added in 7.0
+ return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_256,
+ NewFn);
+ if (Name == "avx512.mask.cmp.ps.512") // Added in 7.0
+ return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_512,
+ NewFn);
// frcz.ss/sd may need to have an argument dropped. Added in 3.2
if (Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) {
@@ -601,6 +633,63 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
return true;
}
}
+
+ // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and v16i8
+ // respectively
+ if ((Name.startswith("arm.neon.bfdot.") ||
+ Name.startswith("aarch64.neon.bfdot.")) &&
+ Name.endswith("i8")) {
+ Intrinsic::ID IID =
+ StringSwitch<Intrinsic::ID>(Name)
+ .Cases("arm.neon.bfdot.v2f32.v8i8",
+ "arm.neon.bfdot.v4f32.v16i8",
+ Intrinsic::arm_neon_bfdot)
+ .Cases("aarch64.neon.bfdot.v2f32.v8i8",
+ "aarch64.neon.bfdot.v4f32.v16i8",
+ Intrinsic::aarch64_neon_bfdot)
+ .Default(Intrinsic::not_intrinsic);
+ if (IID == Intrinsic::not_intrinsic)
+ break;
+
+ size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
+ assert((OperandWidth == 64 || OperandWidth == 128) &&
+ "Unexpected operand width");
+ LLVMContext &Ctx = F->getParent()->getContext();
+ std::array<Type *, 2> Tys {{
+ F->getReturnType(),
+ FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)
+ }};
+ NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
+ return true;
+ }
+
+ // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic anymore
+ // and accept v8bf16 instead of v16i8
+ if ((Name.startswith("arm.neon.bfm") ||
+ Name.startswith("aarch64.neon.bfm")) &&
+ Name.endswith(".v4f32.v16i8")) {
+ Intrinsic::ID IID =
+ StringSwitch<Intrinsic::ID>(Name)
+ .Case("arm.neon.bfmmla.v4f32.v16i8",
+ Intrinsic::arm_neon_bfmmla)
+ .Case("arm.neon.bfmlalb.v4f32.v16i8",
+ Intrinsic::arm_neon_bfmlalb)
+ .Case("arm.neon.bfmlalt.v4f32.v16i8",
+ Intrinsic::arm_neon_bfmlalt)
+ .Case("aarch64.neon.bfmmla.v4f32.v16i8",
+ Intrinsic::aarch64_neon_bfmmla)
+ .Case("aarch64.neon.bfmlalb.v4f32.v16i8",
+ Intrinsic::aarch64_neon_bfmlalb)
+ .Case("aarch64.neon.bfmlalt.v4f32.v16i8",
+ Intrinsic::aarch64_neon_bfmlalt)
+ .Default(Intrinsic::not_intrinsic);
+ if (IID == Intrinsic::not_intrinsic)
+ break;
+
+ std::array<Type *, 0> Tys;
+ NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
+ return true;
+ }
break;
}
@@ -629,18 +718,42 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
}
case 'e': {
SmallVector<StringRef, 2> Groups;
- static const Regex R("^experimental.vector.reduce.([a-z]+)\\.[fi][0-9]+");
+ static const Regex R("^experimental.vector.reduce.([a-z]+)\\.[a-z][0-9]+");
if (R.match(Name, &Groups)) {
+ Intrinsic::ID ID;
+ ID = StringSwitch<Intrinsic::ID>(Groups[1])
+ .Case("add", Intrinsic::vector_reduce_add)
+ .Case("mul", Intrinsic::vector_reduce_mul)
+ .Case("and", Intrinsic::vector_reduce_and)
+ .Case("or", Intrinsic::vector_reduce_or)
+ .Case("xor", Intrinsic::vector_reduce_xor)
+ .Case("smax", Intrinsic::vector_reduce_smax)
+ .Case("smin", Intrinsic::vector_reduce_smin)
+ .Case("umax", Intrinsic::vector_reduce_umax)
+ .Case("umin", Intrinsic::vector_reduce_umin)
+ .Case("fmax", Intrinsic::vector_reduce_fmax)
+ .Case("fmin", Intrinsic::vector_reduce_fmin)
+ .Default(Intrinsic::not_intrinsic);
+ if (ID != Intrinsic::not_intrinsic) {
+ rename(F);
+ auto Args = F->getFunctionType()->params();
+ NewFn = Intrinsic::getDeclaration(F->getParent(), ID, {Args[0]});
+ return true;
+ }
+ }
+ static const Regex R2(
+ "^experimental.vector.reduce.v2.([a-z]+)\\.[fi][0-9]+");
+ Groups.clear();
+ if (R2.match(Name, &Groups)) {
Intrinsic::ID ID = Intrinsic::not_intrinsic;
if (Groups[1] == "fadd")
- ID = Intrinsic::experimental_vector_reduce_v2_fadd;
+ ID = Intrinsic::vector_reduce_fadd;
if (Groups[1] == "fmul")
- ID = Intrinsic::experimental_vector_reduce_v2_fmul;
-
+ ID = Intrinsic::vector_reduce_fmul;
if (ID != Intrinsic::not_intrinsic) {
rename(F);
auto Args = F->getFunctionType()->params();
- Type *Tys[] = {F->getFunctionType()->getReturnType(), Args[1]};
+ Type *Tys[] = {Args[1]};
NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
return true;
}
@@ -900,7 +1013,7 @@ GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
// to byte shuffles.
static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder,
Value *Op, unsigned Shift) {
- auto *ResultTy = cast<VectorType>(Op->getType());
+ auto *ResultTy = cast<FixedVectorType>(Op->getType());
unsigned NumElts = ResultTy->getNumElements() * 8;
// Bitcast from a 64-bit element type to a byte element type.
@@ -934,7 +1047,7 @@ static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder,
// to byte shuffles.
static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
unsigned Shift) {
- auto *ResultTy = cast<VectorType>(Op->getType());
+ auto *ResultTy = cast<FixedVectorType>(Op->getType());
unsigned NumElts = ResultTy->getNumElements() * 8;
// Bitcast from a 64-bit element type to a byte element type.
@@ -966,19 +1079,19 @@ static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
unsigned NumElts) {
+ assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
llvm::VectorType *MaskTy = FixedVectorType::get(
Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
Mask = Builder.CreateBitCast(Mask, MaskTy);
- // If we have less than 8 elements, then the starting mask was an i8 and
- // we need to extract down to the right number of elements.
- if (NumElts < 8) {
+ // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
+ // i8 and we need to extract down to the right number of elements.
+ if (NumElts <= 4) {
int Indices[4];
for (unsigned i = 0; i != NumElts; ++i)
Indices[i] = i;
- Mask = Builder.CreateShuffleVector(Mask, Mask,
- makeArrayRef(Indices, NumElts),
- "extract");
+ Mask = Builder.CreateShuffleVector(
+ Mask, Mask, makeArrayRef(Indices, NumElts), "extract");
}
return Mask;
@@ -992,7 +1105,7 @@ static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask,
return Op0;
Mask = getX86MaskVec(Builder, Mask,
- cast<VectorType>(Op0->getType())->getNumElements());
+ cast<FixedVectorType>(Op0->getType())->getNumElements());
return Builder.CreateSelect(Mask, Op0, Op1);
}
@@ -1019,7 +1132,7 @@ static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
bool IsVALIGN) {
unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
- unsigned NumElts = cast<VectorType>(Op0->getType())->getNumElements();
+ unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
@@ -1120,15 +1233,11 @@ static Value *UpgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallInst &CI,
return EmitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
}
-static Value *UpgradeX86AddSubSatIntrinsics(IRBuilder<> &Builder, CallInst &CI,
- bool IsSigned, bool IsAddition) {
+static Value *UpgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallInst &CI,
+ Intrinsic::ID IID) {
Type *Ty = CI.getType();
Value *Op0 = CI.getOperand(0);
Value *Op1 = CI.getOperand(1);
-
- Intrinsic::ID IID =
- IsSigned ? (IsAddition ? Intrinsic::sadd_sat : Intrinsic::ssub_sat)
- : (IsAddition ? Intrinsic::uadd_sat : Intrinsic::usub_sat);
Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
Value *Res = Builder.CreateCall(Intrin, {Op0, Op1});
@@ -1150,7 +1259,7 @@ static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallInst &CI,
// Funnel shifts amounts are treated as modulo and types are all power-of-2 so
// we only care about the lowest log2 bits anyway.
if (Amt->getType() != Ty) {
- unsigned NumElts = cast<VectorType>(Ty)->getNumElements();
+ unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
Amt = Builder.CreateVectorSplat(NumElts, Amt);
}
@@ -1220,7 +1329,7 @@ static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallInst &CI,
// Funnel shifts amounts are treated as modulo and types are all power-of-2 so
// we only care about the lowest log2 bits anyway.
if (Amt->getType() != Ty) {
- unsigned NumElts = cast<VectorType>(Ty)->getNumElements();
+ unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
Amt = Builder.CreateVectorSplat(NumElts, Amt);
}
@@ -1257,7 +1366,7 @@ static Value *UpgradeMaskedStore(IRBuilder<> &Builder,
return Builder.CreateAlignedStore(Data, Ptr, Alignment);
// Convert the mask from an integer type to a vector of i1.
- unsigned NumElts = cast<VectorType>(Data->getType())->getNumElements();
+ unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
Mask = getX86MaskVec(Builder, Mask, NumElts);
return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
}
@@ -1280,35 +1389,19 @@ static Value *UpgradeMaskedLoad(IRBuilder<> &Builder,
return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
// Convert the mask from an integer type to a vector of i1.
- unsigned NumElts = cast<VectorType>(Passthru->getType())->getNumElements();
+ unsigned NumElts =
+ cast<FixedVectorType>(Passthru->getType())->getNumElements();
Mask = getX86MaskVec(Builder, Mask, NumElts);
return Builder.CreateMaskedLoad(Ptr, Alignment, Mask, Passthru);
}
static Value *upgradeAbs(IRBuilder<> &Builder, CallInst &CI) {
+ Type *Ty = CI.getType();
Value *Op0 = CI.getArgOperand(0);
- llvm::Type *Ty = Op0->getType();
- Value *Zero = llvm::Constant::getNullValue(Ty);
- Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_SGT, Op0, Zero);
- Value *Neg = Builder.CreateNeg(Op0);
- Value *Res = Builder.CreateSelect(Cmp, Op0, Neg);
-
+ Function *F = Intrinsic::getDeclaration(CI.getModule(), Intrinsic::abs, Ty);
+ Value *Res = Builder.CreateCall(F, {Op0, Builder.getInt1(false)});
if (CI.getNumArgOperands() == 3)
- Res = EmitX86Select(Builder,CI.getArgOperand(2), Res, CI.getArgOperand(1));
-
- return Res;
-}
-
-static Value *upgradeIntMinMax(IRBuilder<> &Builder, CallInst &CI,
- ICmpInst::Predicate Pred) {
- Value *Op0 = CI.getArgOperand(0);
- Value *Op1 = CI.getArgOperand(1);
- Value *Cmp = Builder.CreateICmp(Pred, Op0, Op1);
- Value *Res = Builder.CreateSelect(Cmp, Op0, Op1);
-
- if (CI.getNumArgOperands() == 4)
- Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
-
+ Res = EmitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
return Res;
}
@@ -1344,7 +1437,7 @@ static Value *upgradePMULDQ(IRBuilder<> &Builder, CallInst &CI, bool IsSigned) {
// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
Value *Mask) {
- unsigned NumElts = cast<VectorType>(Vec->getType())->getNumElements();
+ unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
if (Mask) {
const auto *C = dyn_cast<Constant>(Mask);
if (!C || !C->isAllOnesValue())
@@ -1367,7 +1460,7 @@ static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI,
unsigned CC, bool Signed) {
Value *Op0 = CI.getArgOperand(0);
- unsigned NumElts = cast<VectorType>(Op0->getType())->getNumElements();
+ unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
Value *Cmp;
if (CC == 3) {
@@ -1422,7 +1515,7 @@ static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallInst &CI) {
static Value* UpgradeMaskToInt(IRBuilder<> &Builder, CallInst &CI) {
Value* Op = CI.getArgOperand(0);
Type* ReturnOp = CI.getType();
- unsigned NumElts = cast<VectorType>(CI.getType())->getNumElements();
+ unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
Value *Mask = getX86MaskVec(Builder, Op, NumElts);
return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
}
@@ -1676,7 +1769,6 @@ void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
(Pos = AsmStr->find("# marker")) != std::string::npos) {
AsmStr->replace(Pos, 1, ";");
}
- return;
}
/// Upgrade a call to an old intrinsic. All argument and return casting must be
@@ -1871,8 +1963,8 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Rep = Builder.CreateICmp(Pred, Rep, Zero);
Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, Mask);
} else if (IsX86 && (Name.startswith("avx512.mask.pbroadcast"))){
- unsigned NumElts =
- cast<VectorType>(CI->getArgOperand(1)->getType())->getNumElements();
+ unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
+ ->getNumElements();
Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
CI->getArgOperand(1));
@@ -2000,38 +2092,36 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
{ CI->getOperand(0), CI->getArgOperand(1) });
Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
- } else if (IsX86 && Name.startswith("avx512.mask.cmp.p")) {
- Type *OpTy = CI->getArgOperand(0)->getType();
+ } else if (IsX86 && Name.startswith("avx512.cmp.p")) {
+ SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
+ CI->arg_operands().end());
+ Type *OpTy = Args[0]->getType();
unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
unsigned EltWidth = OpTy->getScalarSizeInBits();
Intrinsic::ID IID;
if (VecWidth == 128 && EltWidth == 32)
- IID = Intrinsic::x86_avx512_cmp_ps_128;
+ IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
else if (VecWidth == 256 && EltWidth == 32)
- IID = Intrinsic::x86_avx512_cmp_ps_256;
+ IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
else if (VecWidth == 512 && EltWidth == 32)
- IID = Intrinsic::x86_avx512_cmp_ps_512;
+ IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
else if (VecWidth == 128 && EltWidth == 64)
- IID = Intrinsic::x86_avx512_cmp_pd_128;
+ IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
else if (VecWidth == 256 && EltWidth == 64)
- IID = Intrinsic::x86_avx512_cmp_pd_256;
+ IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
else if (VecWidth == 512 && EltWidth == 64)
- IID = Intrinsic::x86_avx512_cmp_pd_512;
+ IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
else
llvm_unreachable("Unexpected intrinsic");
- SmallVector<Value *, 4> Args;
- Args.push_back(CI->getArgOperand(0));
- Args.push_back(CI->getArgOperand(1));
- Args.push_back(CI->getArgOperand(2));
- if (CI->getNumArgOperands() == 5)
- Args.push_back(CI->getArgOperand(4));
+ Value *Mask = Constant::getAllOnesValue(CI->getType());
+ if (VecWidth == 512)
+ std::swap(Mask, Args.back());
+ Args.push_back(Mask);
Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
Args);
- Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(3));
- } else if (IsX86 && Name.startswith("avx512.mask.cmp.") &&
- Name[16] != 'p') {
+ } else if (IsX86 && Name.startswith("avx512.mask.cmp.")) {
// Integer compare intrinsics.
unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
@@ -2057,25 +2147,25 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Name == "sse41.pmaxsd" ||
Name.startswith("avx2.pmaxs") ||
Name.startswith("avx512.mask.pmaxs"))) {
- Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SGT);
+ Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
} else if (IsX86 && (Name == "sse2.pmaxu.b" ||
Name == "sse41.pmaxuw" ||
Name == "sse41.pmaxud" ||
Name.startswith("avx2.pmaxu") ||
Name.startswith("avx512.mask.pmaxu"))) {
- Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_UGT);
+ Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
} else if (IsX86 && (Name == "sse41.pminsb" ||
Name == "sse2.pmins.w" ||
Name == "sse41.pminsd" ||
Name.startswith("avx2.pmins") ||
Name.startswith("avx512.mask.pmins"))) {
- Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SLT);
+ Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
} else if (IsX86 && (Name == "sse2.pminu.b" ||
Name == "sse41.pminuw" ||
Name == "sse41.pminud" ||
Name.startswith("avx2.pminu") ||
Name.startswith("avx512.mask.pminu"))) {
- Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_ULT);
+ Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
} else if (IsX86 && (Name == "sse2.pmulu.dq" ||
Name == "avx2.pmulu.dq" ||
Name == "avx512.pmulu.dq.512" ||
@@ -2122,9 +2212,9 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Name == "avx.cvt.ps2.pd.256" ||
Name == "avx512.mask.cvtps2pd.128" ||
Name == "avx512.mask.cvtps2pd.256")) {
- auto *DstTy = cast<VectorType>(CI->getType());
+ auto *DstTy = cast<FixedVectorType>(CI->getType());
Rep = CI->getArgOperand(0);
- auto *SrcTy = cast<VectorType>(Rep->getType());
+ auto *SrcTy = cast<FixedVectorType>(Rep->getType());
unsigned NumDstElts = DstTy->getNumElements();
if (NumDstElts < SrcTy->getNumElements()) {
@@ -2154,9 +2244,9 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
CI->getArgOperand(1));
} else if (IsX86 && (Name.startswith("avx512.mask.vcvtph2ps.") ||
Name.startswith("vcvtph2ps."))) {
- auto *DstTy = cast<VectorType>(CI->getType());
+ auto *DstTy = cast<FixedVectorType>(CI->getType());
Rep = CI->getArgOperand(0);
- auto *SrcTy = cast<VectorType>(Rep->getType());
+ auto *SrcTy = cast<FixedVectorType>(Rep->getType());
unsigned NumDstElts = DstTy->getNumElements();
if (NumDstElts != SrcTy->getNumElements()) {
assert(NumDstElts == 4 && "Unexpected vector size");
@@ -2177,7 +2267,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
CI->getArgOperand(1),CI->getArgOperand(2),
/*Aligned*/true);
} else if (IsX86 && Name.startswith("avx512.mask.expand.load.")) {
- auto *ResultTy = cast<VectorType>(CI->getType());
+ auto *ResultTy = cast<FixedVectorType>(CI->getType());
Type *PtrTy = ResultTy->getElementType();
// Cast the pointer to element type.
@@ -2199,8 +2289,9 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
llvm::PointerType::getUnqual(PtrTy));
- Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
- ResultTy->getNumElements());
+ Value *MaskVec =
+ getX86MaskVec(Builder, CI->getArgOperand(2),
+ cast<FixedVectorType>(ResultTy)->getNumElements());
Function *CSt = Intrinsic::getDeclaration(F->getParent(),
Intrinsic::masked_compressstore,
@@ -2208,7 +2299,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec });
} else if (IsX86 && (Name.startswith("avx512.mask.compress.") ||
Name.startswith("avx512.mask.expand."))) {
- auto *ResultTy = cast<VectorType>(CI->getType());
+ auto *ResultTy = cast<FixedVectorType>(CI->getType());
Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
ResultTy->getNumElements());
@@ -2288,7 +2379,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
} else if (IsX86 && (Name.startswith("avx.vbroadcast.s") ||
Name.startswith("avx512.vbroadcast.s"))) {
// Replace broadcasts with a series of insertelements.
- auto *VecTy = cast<VectorType>(CI->getType());
+ auto *VecTy = cast<FixedVectorType>(CI->getType());
Type *EltTy = VecTy->getElementType();
unsigned EltNum = VecTy->getNumElements();
Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
@@ -2305,8 +2396,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Name.startswith("avx2.pmovzx") ||
Name.startswith("avx512.mask.pmovsx") ||
Name.startswith("avx512.mask.pmovzx"))) {
- VectorType *SrcTy = cast<VectorType>(CI->getArgOperand(0)->getType());
- VectorType *DstTy = cast<VectorType>(CI->getType());
+ auto *DstTy = cast<FixedVectorType>(CI->getType());
unsigned NumDstElts = DstTy->getNumElements();
// Extract a subvector of the first NumDstElts lanes and sign/zero extend.
@@ -2314,8 +2404,8 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
for (unsigned i = 0; i != NumDstElts; ++i)
ShuffleMask[i] = i;
- Value *SV = Builder.CreateShuffleVector(
- CI->getArgOperand(0), UndefValue::get(SrcTy), ShuffleMask);
+ Value *SV =
+ Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
bool DoSext = (StringRef::npos != Name.find("pmovsx"));
Rep = DoSext ? Builder.CreateSExt(SV, DstTy)
@@ -2342,12 +2432,10 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
PointerType::getUnqual(VT));
Value *Load = Builder.CreateAlignedLoad(VT, Op, Align(1));
if (NumSrcElts == 2)
- Rep = Builder.CreateShuffleVector(
- Load, UndefValue::get(Load->getType()), ArrayRef<int>{0, 1, 0, 1});
+ Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
else
- Rep =
- Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
- ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
+ Rep = Builder.CreateShuffleVector(
+ Load, ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
} else if (IsX86 && (Name.startswith("avx512.mask.shuf.i") ||
Name.startswith("avx512.mask.shuf.f"))) {
unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
@@ -2373,8 +2461,10 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
}else if (IsX86 && (Name.startswith("avx512.mask.broadcastf") ||
Name.startswith("avx512.mask.broadcasti"))) {
unsigned NumSrcElts =
- cast<VectorType>(CI->getArgOperand(0)->getType())->getNumElements();
- unsigned NumDstElts = cast<VectorType>(CI->getType())->getNumElements();
+ cast<FixedVectorType>(CI->getArgOperand(0)->getType())
+ ->getNumElements();
+ unsigned NumDstElts =
+ cast<FixedVectorType>(CI->getType())->getNumElements();
SmallVector<int, 8> ShuffleMask(NumDstElts);
for (unsigned i = 0; i != NumDstElts; ++i)
@@ -2393,30 +2483,31 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Value *Op = CI->getArgOperand(0);
ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
- Rep = Builder.CreateShuffleVector(Op, UndefValue::get(Op->getType()),
- Constant::getNullValue(MaskTy));
+ SmallVector<int, 8> M;
+ ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
+ Rep = Builder.CreateShuffleVector(Op, M);
if (CI->getNumArgOperands() == 3)
Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
CI->getArgOperand(1));
} else if (IsX86 && (Name.startswith("sse2.padds.") ||
- Name.startswith("sse2.psubs.") ||
Name.startswith("avx2.padds.") ||
- Name.startswith("avx2.psubs.") ||
Name.startswith("avx512.padds.") ||
+ Name.startswith("avx512.mask.padds."))) {
+ Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
+ } else if (IsX86 && (Name.startswith("sse2.psubs.") ||
+ Name.startswith("avx2.psubs.") ||
Name.startswith("avx512.psubs.") ||
- Name.startswith("avx512.mask.padds.") ||
Name.startswith("avx512.mask.psubs."))) {
- bool IsAdd = Name.contains(".padds");
- Rep = UpgradeX86AddSubSatIntrinsics(Builder, *CI, true, IsAdd);
+ Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
} else if (IsX86 && (Name.startswith("sse2.paddus.") ||
- Name.startswith("sse2.psubus.") ||
Name.startswith("avx2.paddus.") ||
+ Name.startswith("avx512.mask.paddus."))) {
+ Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
+ } else if (IsX86 && (Name.startswith("sse2.psubus.") ||
Name.startswith("avx2.psubus.") ||
- Name.startswith("avx512.mask.paddus.") ||
Name.startswith("avx512.mask.psubus."))) {
- bool IsAdd = Name.contains(".paddus");
- Rep = UpgradeX86AddSubSatIntrinsics(Builder, *CI, false, IsAdd);
+ Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
} else if (IsX86 && Name.startswith("avx512.mask.palignr.")) {
Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
CI->getArgOperand(1),
@@ -2463,7 +2554,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Value *Op0 = CI->getArgOperand(0);
Value *Op1 = CI->getArgOperand(1);
unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue();
- VectorType *VecTy = cast<VectorType>(CI->getType());
+ auto *VecTy = cast<FixedVectorType>(CI->getType());
unsigned NumElts = VecTy->getNumElements();
SmallVector<int, 16> Idxs(NumElts);
@@ -2477,21 +2568,22 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Value *Op0 = CI->getArgOperand(0);
Value *Op1 = CI->getArgOperand(1);
unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
- unsigned DstNumElts = cast<VectorType>(CI->getType())->getNumElements();
- unsigned SrcNumElts = cast<VectorType>(Op1->getType())->getNumElements();
+ unsigned DstNumElts =
+ cast<FixedVectorType>(CI->getType())->getNumElements();
+ unsigned SrcNumElts =
+ cast<FixedVectorType>(Op1->getType())->getNumElements();
unsigned Scale = DstNumElts / SrcNumElts;
// Mask off the high bits of the immediate value; hardware ignores those.
Imm = Imm % Scale;
// Extend the second operand into a vector the size of the destination.
- Value *UndefV = UndefValue::get(Op1->getType());
SmallVector<int, 8> Idxs(DstNumElts);
for (unsigned i = 0; i != SrcNumElts; ++i)
Idxs[i] = i;
for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
Idxs[i] = SrcNumElts;
- Rep = Builder.CreateShuffleVector(Op1, UndefV, Idxs);
+ Rep = Builder.CreateShuffleVector(Op1, Idxs);
// Insert the second operand into the first operand.
@@ -2521,8 +2613,10 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Name.startswith("avx512.mask.vextract"))) {
Value *Op0 = CI->getArgOperand(0);
unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
- unsigned DstNumElts = cast<VectorType>(CI->getType())->getNumElements();
- unsigned SrcNumElts = cast<VectorType>(Op0->getType())->getNumElements();
+ unsigned DstNumElts =
+ cast<FixedVectorType>(CI->getType())->getNumElements();
+ unsigned SrcNumElts =
+ cast<FixedVectorType>(Op0->getType())->getNumElements();
unsigned Scale = SrcNumElts / DstNumElts;
// Mask off the high bits of the immediate value; hardware ignores those.
@@ -2545,7 +2639,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Name.startswith("avx512.mask.perm.di."))) {
Value *Op0 = CI->getArgOperand(0);
unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
- VectorType *VecTy = cast<VectorType>(CI->getType());
+ auto *VecTy = cast<FixedVectorType>(CI->getType());
unsigned NumElts = VecTy->getNumElements();
SmallVector<int, 8> Idxs(NumElts);
@@ -2569,7 +2663,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
- unsigned NumElts = cast<VectorType>(CI->getType())->getNumElements();
+ unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
unsigned HalfSize = NumElts / 2;
SmallVector<int, 8> ShuffleMask(NumElts);
@@ -2599,7 +2693,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Name.startswith("avx512.mask.pshuf.d."))) {
Value *Op0 = CI->getArgOperand(0);
unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
- VectorType *VecTy = cast<VectorType>(CI->getType());
+ auto *VecTy = cast<FixedVectorType>(CI->getType());
unsigned NumElts = VecTy->getNumElements();
// Calculate the size of each index in the immediate.
unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
@@ -2621,7 +2715,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Name.startswith("avx512.mask.pshufl.w."))) {
Value *Op0 = CI->getArgOperand(0);
unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
- unsigned NumElts = cast<VectorType>(CI->getType())->getNumElements();
+ unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
SmallVector<int, 16> Idxs(NumElts);
for (unsigned l = 0; l != NumElts; l += 8) {
@@ -2640,7 +2734,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Name.startswith("avx512.mask.pshufh.w."))) {
Value *Op0 = CI->getArgOperand(0);
unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
- unsigned NumElts = cast<VectorType>(CI->getType())->getNumElements();
+ unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
SmallVector<int, 16> Idxs(NumElts);
for (unsigned l = 0; l != NumElts; l += 8) {
@@ -2659,7 +2753,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Value *Op0 = CI->getArgOperand(0);
Value *Op1 = CI->getArgOperand(1);
unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
- unsigned NumElts = cast<VectorType>(CI->getType())->getNumElements();
+ unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
unsigned HalfLaneElts = NumLaneElts / 2;
@@ -2684,7 +2778,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Name.startswith("avx512.mask.movshdup") ||
Name.startswith("avx512.mask.movsldup"))) {
Value *Op0 = CI->getArgOperand(0);
- unsigned NumElts = cast<VectorType>(CI->getType())->getNumElements();
+ unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
unsigned Offset = 0;
@@ -2706,7 +2800,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Name.startswith("avx512.mask.unpckl."))) {
Value *Op0 = CI->getArgOperand(0);
Value *Op1 = CI->getArgOperand(1);
- int NumElts = cast<VectorType>(CI->getType())->getNumElements();
+ int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
SmallVector<int, 64> Idxs(NumElts);
@@ -2722,7 +2816,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Name.startswith("avx512.mask.unpckh."))) {
Value *Op0 = CI->getArgOperand(0);
Value *Op1 = CI->getArgOperand(1);
- int NumElts = cast<VectorType>(CI->getType())->getNumElements();
+ int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
SmallVector<int, 64> Idxs(NumElts);
@@ -3290,7 +3384,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
Ops);
} else {
- int NumElts = cast<VectorType>(CI->getType())->getNumElements();
+ int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
CI->getArgOperand(2) };
@@ -3547,28 +3641,6 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
DefaultCase();
return;
}
- case Intrinsic::experimental_vector_reduce_v2_fmul: {
- SmallVector<Value *, 2> Args;
- if (CI->isFast())
- Args.push_back(ConstantFP::get(CI->getOperand(0)->getType(), 1.0));
- else
- Args.push_back(CI->getOperand(0));
- Args.push_back(CI->getOperand(1));
- NewCall = Builder.CreateCall(NewFn, Args);
- cast<Instruction>(NewCall)->copyFastMathFlags(CI);
- break;
- }
- case Intrinsic::experimental_vector_reduce_v2_fadd: {
- SmallVector<Value *, 2> Args;
- if (CI->isFast())
- Args.push_back(Constant::getNullValue(CI->getOperand(0)->getType()));
- else
- Args.push_back(CI->getOperand(0));
- Args.push_back(CI->getOperand(1));
- NewCall = Builder.CreateCall(NewFn, Args);
- cast<Instruction>(NewCall)->copyFastMathFlags(CI);
- break;
- }
case Intrinsic::arm_neon_vld1:
case Intrinsic::arm_neon_vld2:
case Intrinsic::arm_neon_vld3:
@@ -3589,6 +3661,30 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
break;
}
+ case Intrinsic::arm_neon_bfdot:
+ case Intrinsic::arm_neon_bfmmla:
+ case Intrinsic::arm_neon_bfmlalb:
+ case Intrinsic::arm_neon_bfmlalt:
+ case Intrinsic::aarch64_neon_bfdot:
+ case Intrinsic::aarch64_neon_bfmmla:
+ case Intrinsic::aarch64_neon_bfmlalb:
+ case Intrinsic::aarch64_neon_bfmlalt: {
+ SmallVector<Value *, 3> Args;
+ assert(CI->getNumArgOperands() == 3 &&
+ "Mismatch between function args and call args");
+ size_t OperandWidth =
+ CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
+ assert((OperandWidth == 64 || OperandWidth == 128) &&
+ "Unexpected operand width");
+ Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
+ auto Iter = CI->arg_operands().begin();
+ Args.push_back(*Iter++);
+ Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
+ Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
+ NewCall = Builder.CreateCall(NewFn, Args);
+ break;
+ }
+
case Intrinsic::bitreverse:
NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
break;
@@ -3691,11 +3787,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
// Replace the original call result with the first result of the new call.
Value *TSC = Builder.CreateExtractValue(NewCall, 0);
- std::string Name = std::string(CI->getName());
- if (!Name.empty()) {
- CI->setName(Name + ".old");
- NewCall->setName(Name);
- }
+ NewCall->takeName(CI);
CI->replaceAllUsesWith(TSC);
CI->eraseFromParent();
return;
@@ -3718,6 +3810,27 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
break;
}
+ case Intrinsic::x86_avx512_mask_cmp_pd_128:
+ case Intrinsic::x86_avx512_mask_cmp_pd_256:
+ case Intrinsic::x86_avx512_mask_cmp_pd_512:
+ case Intrinsic::x86_avx512_mask_cmp_ps_128:
+ case Intrinsic::x86_avx512_mask_cmp_ps_256:
+ case Intrinsic::x86_avx512_mask_cmp_ps_512: {
+ SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
+ CI->arg_operands().end());
+ unsigned NumElts =
+ cast<FixedVectorType>(Args[0]->getType())->getNumElements();
+ Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
+
+ NewCall = Builder.CreateCall(NewFn, Args);
+ Value *Res = ApplyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
+
+ NewCall->takeName(CI);
+ CI->replaceAllUsesWith(Res);
+ CI->eraseFromParent();
+ return;
+ }
+
case Intrinsic::thread_pointer: {
NewCall = Builder.CreateCall(NewFn, {});
break;
@@ -3766,11 +3879,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
}
assert(NewCall && "Should have either set this variable or returned through "
"the default case");
- std::string Name = std::string(CI->getName());
- if (!Name.empty()) {
- CI->setName(Name + ".old");
- NewCall->setName(Name);
- }
+ NewCall->takeName(CI);
CI->replaceAllUsesWith(NewCall);
CI->eraseFromParent();
}
@@ -3784,8 +3893,8 @@ void llvm::UpgradeCallsToIntrinsic(Function *F) {
if (UpgradeIntrinsicFunction(F, NewFn)) {
// Replace all users of the old function with the new function or new
// instructions. This is not a range loop because the call is deleted.
- for (auto UI = F->user_begin(), UE = F->user_end(); UI != UE; )
- if (CallInst *CI = dyn_cast<CallInst>(*UI++))
+ for (User *U : make_early_inc_range(F->users()))
+ if (CallInst *CI = dyn_cast<CallInst>(U))
UpgradeIntrinsicCall(CI, NewFn);
// Remove old function, no longer used, from the module.
@@ -3921,8 +4030,8 @@ void llvm::UpgradeARCRuntime(Module &M) {
Function *NewFn = llvm::Intrinsic::getDeclaration(&M, IntrinsicFunc);
- for (auto I = Fn->user_begin(), E = Fn->user_end(); I != E;) {
- CallInst *CI = dyn_cast<CallInst>(*I++);
+ for (User *U : make_early_inc_range(Fn->users())) {
+ CallInst *CI = dyn_cast<CallInst>(U);
if (!CI || CI->getCalledFunction() != Fn)
continue;
@@ -3963,7 +4072,7 @@ void llvm::UpgradeARCRuntime(Module &M) {
// Create a call instruction that calls the new function.
CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
- NewCall->setName(CI->getName());
+ NewCall->takeName(CI);
// Bitcast the return value back to the type of the old call.
Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
@@ -4202,6 +4311,13 @@ void llvm::UpgradeFunctionAttributes(Function &F) {
StrictFPUpgradeVisitor SFPV;
SFPV.visit(F);
}
+
+ if (F.getCallingConv() == CallingConv::X86_INTR &&
+ !F.arg_empty() && !F.hasParamAttribute(0, Attribute::ByVal)) {
+ Type *ByValTy = cast<PointerType>(F.getArg(0)->getType())->getElementType();
+ Attribute NewAttr = Attribute::getWithByValType(F.getContext(), ByValTy);
+ F.addParamAttr(0, NewAttr);
+ }
}
static bool isOldLoopArgument(Metadata *MD) {
@@ -4267,11 +4383,17 @@ MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
}
std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
- std::string AddrSpaces = "-p270:32:32-p271:32:32-p272:64:64";
+ Triple T(TT);
+ // For AMDGPU we uprgrade older DataLayouts to include the default globals
+ // address space of 1.
+ if (T.isAMDGPU() && !DL.contains("-G") && !DL.startswith("G")) {
+ return DL.empty() ? std::string("G1") : (DL + "-G1").str();
+ }
+ std::string AddrSpaces = "-p270:32:32-p271:32:32-p272:64:64";
// If X86, and the datalayout matches the expected format, add pointer size
// address spaces to the datalayout.
- if (!Triple(TT).isX86() || DL.contains(AddrSpaces))
+ if (!T.isX86() || DL.contains(AddrSpaces))
return std::string(DL);
SmallVector<StringRef, 4> Groups;
@@ -4279,9 +4401,7 @@ std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
if (!R.match(DL, &Groups))
return std::string(DL);
- SmallString<1024> Buf;
- std::string Res = (Groups[1] + AddrSpaces + Groups[3]).toStringRef(Buf).str();
- return Res;
+ return (Groups[1] + AddrSpaces + Groups[3]).str();
}
void llvm::UpgradeAttributes(AttrBuilder &B) {
diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp
index 64f1d3f3100c..00ef10dd53af 100644
--- a/llvm/lib/IR/BasicBlock.cpp
+++ b/llvm/lib/IR/BasicBlock.cpp
@@ -97,18 +97,20 @@ void BasicBlock::setParent(Function *parent) {
iterator_range<filter_iterator<BasicBlock::const_iterator,
std::function<bool(const Instruction &)>>>
-BasicBlock::instructionsWithoutDebug() const {
- std::function<bool(const Instruction &)> Fn = [](const Instruction &I) {
- return !isa<DbgInfoIntrinsic>(I);
+BasicBlock::instructionsWithoutDebug(bool SkipPseudoOp) const {
+ std::function<bool(const Instruction &)> Fn = [=](const Instruction &I) {
+ return !isa<DbgInfoIntrinsic>(I) &&
+ !(SkipPseudoOp && isa<PseudoProbeInst>(I));
};
return make_filter_range(*this, Fn);
}
-iterator_range<filter_iterator<BasicBlock::iterator,
- std::function<bool(Instruction &)>>>
-BasicBlock::instructionsWithoutDebug() {
- std::function<bool(Instruction &)> Fn = [](Instruction &I) {
- return !isa<DbgInfoIntrinsic>(I);
+iterator_range<
+ filter_iterator<BasicBlock::iterator, std::function<bool(Instruction &)>>>
+BasicBlock::instructionsWithoutDebug(bool SkipPseudoOp) {
+ std::function<bool(Instruction &)> Fn = [=](Instruction &I) {
+ return !isa<DbgInfoIntrinsic>(I) &&
+ !(SkipPseudoOp && isa<PseudoProbeInst>(I));
};
return make_filter_range(*this, Fn);
}
@@ -128,15 +130,11 @@ iplist<BasicBlock>::iterator BasicBlock::eraseFromParent() {
return getParent()->getBasicBlockList().erase(getIterator());
}
-/// Unlink this basic block from its current function and
-/// insert it into the function that MovePos lives in, right before MovePos.
void BasicBlock::moveBefore(BasicBlock *MovePos) {
MovePos->getParent()->getBasicBlockList().splice(
MovePos->getIterator(), getParent()->getBasicBlockList(), getIterator());
}
-/// Unlink this basic block from its current function and
-/// insert it into the function that MovePos lives in, right after MovePos.
void BasicBlock::moveAfter(BasicBlock *MovePos) {
MovePos->getParent()->getBasicBlockList().splice(
++MovePos->getIterator(), getParent()->getBasicBlockList(),
@@ -218,14 +216,21 @@ const Instruction* BasicBlock::getFirstNonPHI() const {
return nullptr;
}
-const Instruction* BasicBlock::getFirstNonPHIOrDbg() const {
- for (const Instruction &I : *this)
- if (!isa<PHINode>(I) && !isa<DbgInfoIntrinsic>(I))
- return &I;
+const Instruction *BasicBlock::getFirstNonPHIOrDbg(bool SkipPseudoOp) const {
+ for (const Instruction &I : *this) {
+ if (isa<PHINode>(I) || isa<DbgInfoIntrinsic>(I))
+ continue;
+
+ if (SkipPseudoOp && isa<PseudoProbeInst>(I))
+ continue;
+
+ return &I;
+ }
return nullptr;
}
-const Instruction* BasicBlock::getFirstNonPHIOrDbgOrLifetime() const {
+const Instruction *
+BasicBlock::getFirstNonPHIOrDbgOrLifetime(bool SkipPseudoOp) const {
for (const Instruction &I : *this) {
if (isa<PHINode>(I) || isa<DbgInfoIntrinsic>(I))
continue;
@@ -233,6 +238,9 @@ const Instruction* BasicBlock::getFirstNonPHIOrDbgOrLifetime() const {
if (I.isLifetimeStartOrEnd())
continue;
+ if (SkipPseudoOp && isa<PseudoProbeInst>(I))
+ continue;
+
return &I;
}
return nullptr;
@@ -253,8 +261,6 @@ void BasicBlock::dropAllReferences() {
I.dropAllReferences();
}
-/// If this basic block has a single predecessor block,
-/// return the block, otherwise return a null pointer.
const BasicBlock *BasicBlock::getSinglePredecessor() const {
const_pred_iterator PI = pred_begin(this), E = pred_end(this);
if (PI == E) return nullptr; // No preds.
@@ -263,11 +269,6 @@ const BasicBlock *BasicBlock::getSinglePredecessor() const {
return (PI == E) ? ThePred : nullptr /*multiple preds*/;
}
-/// If this basic block has a unique predecessor block,
-/// return the block, otherwise return a null pointer.
-/// Note that unique predecessor doesn't mean single edge, there can be
-/// multiple edges from the unique predecessor to this block (for example
-/// a switch statement with multiple cases having the same destination).
const BasicBlock *BasicBlock::getUniquePredecessor() const {
const_pred_iterator PI = pred_begin(this), E = pred_end(this);
if (PI == E) return nullptr; // No preds.
@@ -317,38 +318,31 @@ iterator_range<BasicBlock::phi_iterator> BasicBlock::phis() {
return make_range<phi_iterator>(P, nullptr);
}
-/// Update PHI nodes in this BasicBlock before removal of predecessor \p Pred.
-/// Note that this function does not actually remove the predecessor.
-///
-/// If \p KeepOneInputPHIs is true then don't remove PHIs that are left with
-/// zero or one incoming values, and don't simplify PHIs with all incoming
-/// values the same.
void BasicBlock::removePredecessor(BasicBlock *Pred,
bool KeepOneInputPHIs) {
// Use hasNUsesOrMore to bound the cost of this assertion for complex CFGs.
- assert((hasNUsesOrMore(16) ||
- find(pred_begin(this), pred_end(this), Pred) != pred_end(this)) &&
+ assert((hasNUsesOrMore(16) || llvm::is_contained(predecessors(this), Pred)) &&
"Pred is not a predecessor!");
// Return early if there are no PHI nodes to update.
- if (!isa<PHINode>(begin()))
+ if (empty() || !isa<PHINode>(begin()))
return;
+
unsigned NumPreds = cast<PHINode>(front()).getNumIncomingValues();
+ for (PHINode &Phi : make_early_inc_range(phis())) {
+ Phi.removeIncomingValue(Pred, !KeepOneInputPHIs);
+ if (KeepOneInputPHIs)
+ continue;
+
+ // If we have a single predecessor, removeIncomingValue may have erased the
+ // PHI node itself.
+ if (NumPreds == 1)
+ continue;
- // Update all PHI nodes.
- for (iterator II = begin(); isa<PHINode>(II);) {
- PHINode *PN = cast<PHINode>(II++);
- PN->removeIncomingValue(Pred, !KeepOneInputPHIs);
- if (!KeepOneInputPHIs) {
- // If we have a single predecessor, removeIncomingValue erased the PHI
- // node itself.
- if (NumPreds > 1) {
- if (Value *PNV = PN->hasConstantValue()) {
- // Replace the PHI node with its constant value.
- PN->replaceAllUsesWith(PNV);
- PN->eraseFromParent();
- }
- }
+ // Try to replace the PHI node with a constant value.
+ if (Value *PhiConstant = Phi.hasConstantValue()) {
+ Phi.replaceAllUsesWith(PhiConstant);
+ Phi.eraseFromParent();
}
}
}
@@ -378,18 +372,11 @@ bool BasicBlock::isLegalToHoistInto() const {
return !Term->isExceptionalTerminator();
}
-/// This splits a basic block into two at the specified
-/// instruction. Note that all instructions BEFORE the specified iterator stay
-/// as part of the original basic block, an unconditional branch is added to
-/// the new BB, and the rest of the instructions in the BB are moved to the new
-/// BB, including the old terminator. This invalidates the iterator.
-///
-/// Note that this only works on well formed basic blocks (must have a
-/// terminator), and 'I' must not be the end of instruction list (which would
-/// cause a degenerate basic block to be formed, having a terminator inside of
-/// the basic block).
-///
-BasicBlock *BasicBlock::splitBasicBlock(iterator I, const Twine &BBName) {
+BasicBlock *BasicBlock::splitBasicBlock(iterator I, const Twine &BBName,
+ bool Before) {
+ if (Before)
+ return splitBasicBlockBefore(I, BBName);
+
assert(getTerminator() && "Can't use splitBasicBlock on degenerate BB!");
assert(I != InstList.end() &&
"Trying to get me to create degenerate basic block!");
@@ -416,6 +403,40 @@ BasicBlock *BasicBlock::splitBasicBlock(iterator I, const Twine &BBName) {
return New;
}
+BasicBlock *BasicBlock::splitBasicBlockBefore(iterator I, const Twine &BBName) {
+ assert(getTerminator() &&
+ "Can't use splitBasicBlockBefore on degenerate BB!");
+ assert(I != InstList.end() &&
+ "Trying to get me to create degenerate basic block!");
+
+ assert((!isa<PHINode>(*I) || getSinglePredecessor()) &&
+ "cannot split on multi incoming phis");
+
+ BasicBlock *New = BasicBlock::Create(getContext(), BBName, getParent(), this);
+ // Save DebugLoc of split point before invalidating iterator.
+ DebugLoc Loc = I->getDebugLoc();
+ // Move all of the specified instructions from the original basic block into
+ // the new basic block.
+ New->getInstList().splice(New->end(), this->getInstList(), begin(), I);
+
+ // Loop through all of the predecessors of the 'this' block (which will be the
+ // predecessors of the New block), replace the specified successor 'this'
+ // block to point at the New block and update any PHI nodes in 'this' block.
+ // If there were PHI nodes in 'this' block, the PHI nodes are updated
+ // to reflect that the incoming branches will be from the New block and not
+ // from predecessors of the 'this' block.
+ for (BasicBlock *Pred : predecessors(this)) {
+ Instruction *TI = Pred->getTerminator();
+ TI->replaceSuccessorWith(this, New);
+ this->replacePhiUsesWith(Pred, New);
+ }
+ // Add a branch instruction from "New" to "this" Block.
+ BranchInst *BI = BranchInst::Create(this, New);
+ BI->setDebugLoc(Loc);
+
+ return New;
+}
+
void BasicBlock::replacePhiUsesWith(BasicBlock *Old, BasicBlock *New) {
// N.B. This might not be a complete BasicBlock, so don't assume
// that it ends with a non-phi instruction.
@@ -443,13 +464,10 @@ void BasicBlock::replaceSuccessorsPhiUsesWith(BasicBlock *New) {
this->replaceSuccessorsPhiUsesWith(this, New);
}
-/// Return true if this basic block is a landing pad. I.e., it's
-/// the destination of the 'unwind' edge of an invoke instruction.
bool BasicBlock::isLandingPad() const {
return isa<LandingPadInst>(getFirstNonPHI());
}
-/// Return the landingpad instruction associated with the landing pad.
const LandingPadInst *BasicBlock::getLandingPadInst() const {
return dyn_cast<LandingPadInst>(getFirstNonPHI());
}
diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp
index f02246cda7fc..03cb108cc485 100644
--- a/llvm/lib/IR/ConstantFold.cpp
+++ b/llvm/lib/IR/ConstantFold.cpp
@@ -522,6 +522,9 @@ static Constant *getFoldedOffsetOf(Type *Ty, Constant *FieldNo, Type *DestTy,
Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V,
Type *DestTy) {
+ if (isa<PoisonValue>(V))
+ return PoisonValue::get(DestTy);
+
if (isa<UndefValue>(V)) {
// zext(undef) = 0, because the top bits will be zero.
// sext(undef) = 0, because the top bits will all be the same.
@@ -532,7 +535,7 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V,
return UndefValue::get(DestTy);
}
- if (V->isNullValue() && !DestTy->isX86_MMXTy() &&
+ if (V->isNullValue() && !DestTy->isX86_MMXTy() && !DestTy->isX86_AMXTy() &&
opc != Instruction::AddrSpaceCast)
return Constant::getNullValue(DestTy);
@@ -627,7 +630,7 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V,
V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored)) {
// Undefined behavior invoked - the destination type can't represent
// the input constant.
- return UndefValue::get(DestTy);
+ return PoisonValue::get(DestTy);
}
return ConstantInt::get(FPC->getContext(), IntVal);
}
@@ -759,7 +762,9 @@ Constant *llvm::ConstantFoldSelectInstruction(Constant *Cond,
Constant *V2Element = ConstantExpr::getExtractElement(V2,
ConstantInt::get(Ty, i));
auto *Cond = cast<Constant>(CondV->getOperand(i));
- if (V1Element == V2Element) {
+ if (isa<PoisonValue>(Cond)) {
+ V = PoisonValue::get(V1Element->getType());
+ } else if (V1Element == V2Element) {
V = V1Element;
} else if (isa<UndefValue>(Cond)) {
V = isa<UndefValue>(V1Element) ? V1Element : V2Element;
@@ -775,6 +780,9 @@ Constant *llvm::ConstantFoldSelectInstruction(Constant *Cond,
return ConstantVector::get(Result);
}
+ if (isa<PoisonValue>(Cond))
+ return PoisonValue::get(V1->getType());
+
if (isa<UndefValue>(Cond)) {
if (isa<UndefValue>(V1)) return V1;
return V2;
@@ -782,9 +790,17 @@ Constant *llvm::ConstantFoldSelectInstruction(Constant *Cond,
if (V1 == V2) return V1;
+ if (isa<PoisonValue>(V1))
+ return V2;
+ if (isa<PoisonValue>(V2))
+ return V1;
+
// If the true or false value is undef, we can fold to the other value as
// long as the other value isn't poison.
auto NotPoison = [](Constant *C) {
+ if (isa<PoisonValue>(C))
+ return false;
+
// TODO: We can analyze ConstExpr by opcode to determine if there is any
// possibility of poison.
if (isa<ConstantExpr>(C))
@@ -795,7 +811,7 @@ Constant *llvm::ConstantFoldSelectInstruction(Constant *Cond,
return true;
if (C->getType()->isVectorTy())
- return !C->containsUndefElement() && !C->containsConstantExpression();
+ return !C->containsPoisonElement() && !C->containsConstantExpression();
// TODO: Recursively analyze aggregates or other constants.
return false;
@@ -821,9 +837,13 @@ Constant *llvm::ConstantFoldExtractElementInstruction(Constant *Val,
Constant *Idx) {
auto *ValVTy = cast<VectorType>(Val->getType());
+ // extractelt poison, C -> poison
+ // extractelt C, undef -> poison
+ if (isa<PoisonValue>(Val) || isa<UndefValue>(Idx))
+ return PoisonValue::get(ValVTy->getElementType());
+
// extractelt undef, C -> undef
- // extractelt C, undef -> undef
- if (isa<UndefValue>(Val) || isa<UndefValue>(Idx))
+ if (isa<UndefValue>(Val))
return UndefValue::get(ValVTy->getElementType());
auto *CIdx = dyn_cast<ConstantInt>(Idx);
@@ -831,9 +851,9 @@ Constant *llvm::ConstantFoldExtractElementInstruction(Constant *Val,
return nullptr;
if (auto *ValFVTy = dyn_cast<FixedVectorType>(Val->getType())) {
- // ee({w,x,y,z}, wrong_value) -> undef
+ // ee({w,x,y,z}, wrong_value) -> poison
if (CIdx->uge(ValFVTy->getNumElements()))
- return UndefValue::get(ValFVTy->getElementType());
+ return PoisonValue::get(ValFVTy->getElementType());
}
// ee (gep (ptr, idx0, ...), idx) -> gep (ee (ptr, idx), ee (idx0, idx), ...)
@@ -853,6 +873,15 @@ Constant *llvm::ConstantFoldExtractElementInstruction(Constant *Val,
}
return CE->getWithOperands(Ops, ValVTy->getElementType(), false,
Ops[0]->getType()->getPointerElementType());
+ } else if (CE->getOpcode() == Instruction::InsertElement) {
+ if (const auto *IEIdx = dyn_cast<ConstantInt>(CE->getOperand(2))) {
+ if (APSInt::isSameValue(APSInt(IEIdx->getValue()),
+ APSInt(CIdx->getValue()))) {
+ return CE->getOperand(1);
+ } else {
+ return ConstantExpr::getExtractElement(CE->getOperand(0), CIdx);
+ }
+ }
}
}
@@ -873,7 +902,7 @@ Constant *llvm::ConstantFoldInsertElementInstruction(Constant *Val,
Constant *Elt,
Constant *Idx) {
if (isa<UndefValue>(Idx))
- return UndefValue::get(Val->getType());
+ return PoisonValue::get(Val->getType());
ConstantInt *CIdx = dyn_cast<ConstantInt>(Idx);
if (!CIdx) return nullptr;
@@ -887,7 +916,7 @@ Constant *llvm::ConstantFoldInsertElementInstruction(Constant *Val,
unsigned NumElts = ValTy->getNumElements();
if (CIdx->uge(NumElts))
- return UndefValue::get(Val->getType());
+ return PoisonValue::get(Val->getType());
SmallVector<Constant*, 16> Result;
Result.reserve(NumElts);
@@ -910,7 +939,8 @@ Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
ArrayRef<int> Mask) {
auto *V1VTy = cast<VectorType>(V1->getType());
unsigned MaskNumElts = Mask.size();
- ElementCount MaskEltCount = {MaskNumElts, isa<ScalableVectorType>(V1VTy)};
+ auto MaskEltCount =
+ ElementCount::get(MaskNumElts, isa<ScalableVectorType>(V1VTy));
Type *EltTy = V1VTy->getElementType();
// Undefined shuffle mask -> undefined value.
@@ -921,7 +951,7 @@ Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
// If the mask is all zeros this is a splat, no need to go through all
// elements.
if (all_of(Mask, [](int Elt) { return Elt == 0; }) &&
- !MaskEltCount.Scalable) {
+ !MaskEltCount.isScalable()) {
Type *Ty = IntegerType::get(V1->getContext(), 32);
Constant *Elt =
ConstantExpr::getExtractElement(V1, ConstantInt::get(Ty, 0));
@@ -932,7 +962,7 @@ Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
if (isa<ScalableVectorType>(V1VTy))
return nullptr;
- unsigned SrcNumElts = V1VTy->getElementCount().Min;
+ unsigned SrcNumElts = V1VTy->getElementCount().getKnownMinValue();
// Loop over the shuffle mask, evaluating each element.
SmallVector<Constant*, 32> Result;
@@ -1074,6 +1104,17 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1,
return C1;
}
+ // Binary operations propagate poison.
+ // FIXME: Currently, or/and i1 poison aren't folded into poison because
+ // it causes miscompilation when combined with another optimization in
+ // InstCombine (select i1 -> and/or). The select fold is wrong, but
+ // fixing it requires an effort, so temporarily disable this until it is
+ // fixed.
+ bool PoisonFold = !C1->getType()->isIntegerTy(1) ||
+ (Opcode != Instruction::Or && Opcode != Instruction::And);
+ if (PoisonFold && (isa<PoisonValue>(C1) || isa<PoisonValue>(C2)))
+ return PoisonValue::get(C1->getType());
+
// Handle scalar UndefValue and scalable vector UndefValue. Fixed-length
// vectors are always evaluated per element.
bool IsScalableVector = isa<ScalableVectorType>(C1->getType());
@@ -1110,23 +1151,21 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1,
}
case Instruction::SDiv:
case Instruction::UDiv:
- // X / undef -> undef
- if (isa<UndefValue>(C2))
- return C2;
- // undef / 0 -> undef
+ // X / undef -> poison
+ // X / 0 -> poison
+ if (match(C2, m_CombineOr(m_Undef(), m_Zero())))
+ return PoisonValue::get(C2->getType());
// undef / 1 -> undef
- if (match(C2, m_Zero()) || match(C2, m_One()))
+ if (match(C2, m_One()))
return C1;
// undef / X -> 0 otherwise
return Constant::getNullValue(C1->getType());
case Instruction::URem:
case Instruction::SRem:
- // X % undef -> undef
- if (match(C2, m_Undef()))
- return C2;
- // undef % 0 -> undef
- if (match(C2, m_Zero()))
- return C1;
+ // X % undef -> poison
+ // X % 0 -> poison
+ if (match(C2, m_CombineOr(m_Undef(), m_Zero())))
+ return PoisonValue::get(C2->getType());
// undef % X -> 0 otherwise
return Constant::getNullValue(C1->getType());
case Instruction::Or: // X | undef -> -1
@@ -1134,28 +1173,28 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1,
return C1;
return Constant::getAllOnesValue(C1->getType()); // undef | X -> ~0
case Instruction::LShr:
- // X >>l undef -> undef
+ // X >>l undef -> poison
if (isa<UndefValue>(C2))
- return C2;
+ return PoisonValue::get(C2->getType());
// undef >>l 0 -> undef
if (match(C2, m_Zero()))
return C1;
// undef >>l X -> 0
return Constant::getNullValue(C1->getType());
case Instruction::AShr:
- // X >>a undef -> undef
+ // X >>a undef -> poison
if (isa<UndefValue>(C2))
- return C2;
+ return PoisonValue::get(C2->getType());
// undef >>a 0 -> undef
if (match(C2, m_Zero()))
return C1;
- // TODO: undef >>a X -> undef if the shift is exact
+ // TODO: undef >>a X -> poison if the shift is exact
// undef >>a X -> 0
return Constant::getNullValue(C1->getType());
case Instruction::Shl:
// X << undef -> undef
if (isa<UndefValue>(C2))
- return C2;
+ return PoisonValue::get(C2->getType());
// undef << 0 -> undef
if (match(C2, m_Zero()))
return C1;
@@ -1208,14 +1247,14 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1,
if (CI2->isOne())
return C1; // X / 1 == X
if (CI2->isZero())
- return UndefValue::get(CI2->getType()); // X / 0 == undef
+ return PoisonValue::get(CI2->getType()); // X / 0 == poison
break;
case Instruction::URem:
case Instruction::SRem:
if (CI2->isOne())
return Constant::getNullValue(CI2->getType()); // X % 1 == 0
if (CI2->isZero())
- return UndefValue::get(CI2->getType()); // X % 0 == undef
+ return PoisonValue::get(CI2->getType()); // X % 0 == poison
break;
case Instruction::And:
if (CI2->isZero()) return C2; // X & 0 == 0
@@ -1329,7 +1368,7 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1,
case Instruction::SDiv:
assert(!CI2->isZero() && "Div by zero handled above");
if (C2V.isAllOnesValue() && C1V.isMinSignedValue())
- return UndefValue::get(CI1->getType()); // MIN_INT / -1 -> undef
+ return PoisonValue::get(CI1->getType()); // MIN_INT / -1 -> poison
return ConstantInt::get(CI1->getContext(), C1V.sdiv(C2V));
case Instruction::URem:
assert(!CI2->isZero() && "Div by zero handled above");
@@ -1337,7 +1376,7 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1,
case Instruction::SRem:
assert(!CI2->isZero() && "Div by zero handled above");
if (C2V.isAllOnesValue() && C1V.isMinSignedValue())
- return UndefValue::get(CI1->getType()); // MIN_INT % -1 -> undef
+ return PoisonValue::get(CI1->getType()); // MIN_INT % -1 -> poison
return ConstantInt::get(CI1->getContext(), C1V.srem(C2V));
case Instruction::And:
return ConstantInt::get(CI1->getContext(), C1V & C2V);
@@ -1348,15 +1387,15 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1,
case Instruction::Shl:
if (C2V.ult(C1V.getBitWidth()))
return ConstantInt::get(CI1->getContext(), C1V.shl(C2V));
- return UndefValue::get(C1->getType()); // too big shift is undef
+ return PoisonValue::get(C1->getType()); // too big shift is poison
case Instruction::LShr:
if (C2V.ult(C1V.getBitWidth()))
return ConstantInt::get(CI1->getContext(), C1V.lshr(C2V));
- return UndefValue::get(C1->getType()); // too big shift is undef
+ return PoisonValue::get(C1->getType()); // too big shift is poison
case Instruction::AShr:
if (C2V.ult(C1V.getBitWidth()))
return ConstantInt::get(CI1->getContext(), C1V.ashr(C2V));
- return UndefValue::get(C1->getType()); // too big shift is undef
+ return PoisonValue::get(C1->getType()); // too big shift is poison
}
}
@@ -1398,16 +1437,11 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1,
return ConstantFP::get(C1->getContext(), C3V);
}
}
- } else if (IsScalableVector) {
- // Do not iterate on scalable vector. The number of elements is unknown at
- // compile-time.
- // FIXME: this branch can potentially be removed
- return nullptr;
- } else if (auto *VTy = dyn_cast<FixedVectorType>(C1->getType())) {
+ } else if (auto *VTy = dyn_cast<VectorType>(C1->getType())) {
// Fast path for splatted constants.
if (Constant *C2Splat = C2->getSplatValue()) {
if (Instruction::isIntDivRem(Opcode) && C2Splat->isNullValue())
- return UndefValue::get(VTy);
+ return PoisonValue::get(VTy);
if (Constant *C1Splat = C1->getSplatValue()) {
return ConstantVector::getSplat(
VTy->getElementCount(),
@@ -1415,22 +1449,24 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1,
}
}
- // Fold each element and create a vector constant from those constants.
- SmallVector<Constant*, 16> Result;
- Type *Ty = IntegerType::get(VTy->getContext(), 32);
- for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) {
- Constant *ExtractIdx = ConstantInt::get(Ty, i);
- Constant *LHS = ConstantExpr::getExtractElement(C1, ExtractIdx);
- Constant *RHS = ConstantExpr::getExtractElement(C2, ExtractIdx);
+ if (auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
+ // Fold each element and create a vector constant from those constants.
+ SmallVector<Constant*, 16> Result;
+ Type *Ty = IntegerType::get(FVTy->getContext(), 32);
+ for (unsigned i = 0, e = FVTy->getNumElements(); i != e; ++i) {
+ Constant *ExtractIdx = ConstantInt::get(Ty, i);
+ Constant *LHS = ConstantExpr::getExtractElement(C1, ExtractIdx);
+ Constant *RHS = ConstantExpr::getExtractElement(C2, ExtractIdx);
- // If any element of a divisor vector is zero, the whole op is undef.
- if (Instruction::isIntDivRem(Opcode) && RHS->isNullValue())
- return UndefValue::get(VTy);
+ // If any element of a divisor vector is zero, the whole op is poison.
+ if (Instruction::isIntDivRem(Opcode) && RHS->isNullValue())
+ return PoisonValue::get(VTy);
- Result.push_back(ConstantExpr::get(Opcode, LHS, RHS));
- }
+ Result.push_back(ConstantExpr::get(Opcode, LHS, RHS));
+ }
- return ConstantVector::get(Result);
+ return ConstantVector::get(Result);
+ }
}
if (ConstantExpr *CE1 = dyn_cast<ConstantExpr>(C1)) {
@@ -1609,7 +1645,7 @@ static FCmpInst::Predicate evaluateFCmpRelation(Constant *V1, Constant *V2) {
static ICmpInst::Predicate areGlobalsPotentiallyEqual(const GlobalValue *GV1,
const GlobalValue *GV2) {
auto isGlobalUnsafeForEquality = [](const GlobalValue *GV) {
- if (GV->hasExternalWeakLinkage() || GV->hasWeakAnyLinkage())
+ if (GV->isInterposable() || GV->hasGlobalUnnamedAddr())
return true;
if (const auto *GVar = dyn_cast<GlobalVariable>(GV)) {
Type *Ty = GVar->getValueType();
@@ -1744,9 +1780,15 @@ static ICmpInst::Predicate evaluateICmpRelation(Constant *V1, Constant *V2,
case Instruction::FPToSI:
break; // We can't evaluate floating point casts or truncations.
+ case Instruction::BitCast:
+ // If this is a global value cast, check to see if the RHS is also a
+ // GlobalValue.
+ if (const GlobalValue *GV = dyn_cast<GlobalValue>(CE1Op0))
+ if (const GlobalValue *GV2 = dyn_cast<GlobalValue>(V2))
+ return areGlobalsPotentiallyEqual(GV, GV2);
+ LLVM_FALLTHROUGH;
case Instruction::UIToFP:
case Instruction::SIToFP:
- case Instruction::BitCast:
case Instruction::ZExt:
case Instruction::SExt:
// We can't evaluate floating point casts or truncations.
@@ -1909,6 +1951,9 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred,
return Constant::getAllOnesValue(ResultTy);
// Handle some degenerate cases first
+ if (isa<PoisonValue>(C1) || isa<PoisonValue>(C2))
+ return PoisonValue::get(ResultTy);
+
if (isa<UndefValue>(C1) || isa<UndefValue>(C2)) {
CmpInst::Predicate Predicate = CmpInst::Predicate(pred);
bool isIntegerPredicate = ICmpInst::isIntPredicate(Predicate);
@@ -2029,11 +2074,6 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred,
}
} else if (auto *C1VTy = dyn_cast<VectorType>(C1->getType())) {
- // Do not iterate on scalable vector. The number of elements is unknown at
- // compile-time.
- if (isa<ScalableVectorType>(C1VTy))
- return nullptr;
-
// Fast path for splatted constants.
if (Constant *C1Splat = C1->getSplatValue())
if (Constant *C2Splat = C2->getSplatValue())
@@ -2041,16 +2081,22 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred,
C1VTy->getElementCount(),
ConstantExpr::getCompare(pred, C1Splat, C2Splat));
+ // Do not iterate on scalable vector. The number of elements is unknown at
+ // compile-time.
+ if (isa<ScalableVectorType>(C1VTy))
+ return nullptr;
+
// If we can constant fold the comparison of each element, constant fold
// the whole vector comparison.
SmallVector<Constant*, 4> ResElts;
Type *Ty = IntegerType::get(C1->getContext(), 32);
// Compare the elements, producing an i1 result or constant expr.
- for (unsigned i = 0, e = C1VTy->getElementCount().Min; i != e; ++i) {
+ for (unsigned I = 0, E = C1VTy->getElementCount().getKnownMinValue();
+ I != E; ++I) {
Constant *C1E =
- ConstantExpr::getExtractElement(C1, ConstantInt::get(Ty, i));
+ ConstantExpr::getExtractElement(C1, ConstantInt::get(Ty, I));
Constant *C2E =
- ConstantExpr::getExtractElement(C2, ConstantInt::get(Ty, i));
+ ConstantExpr::getExtractElement(C2, ConstantInt::get(Ty, I));
ResElts.push_back(ConstantExpr::getCompare(pred, C1E, C2E));
}
@@ -2293,8 +2339,12 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C,
Type *GEPTy = GetElementPtrInst::getGEPReturnType(
PointeeTy, C, makeArrayRef((Value *const *)Idxs.data(), Idxs.size()));
+ if (isa<PoisonValue>(C))
+ return PoisonValue::get(GEPTy);
+
if (isa<UndefValue>(C))
- return UndefValue::get(GEPTy);
+ // If inbounds, we can choose an out-of-bounds pointer as a base pointer.
+ return InBounds ? PoisonValue::get(GEPTy) : UndefValue::get(GEPTy);
Constant *Idx0 = cast<Constant>(Idxs[0]);
if (Idxs.size() == 1 && (Idx0->isNullValue() || isa<UndefValue>(Idx0)))
diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp
index eabaaa203927..4b0ad1bd25a0 100644
--- a/llvm/lib/IR/ConstantRange.cpp
+++ b/llvm/lib/IR/ConstantRange.cpp
@@ -26,6 +26,7 @@
#include "llvm/IR/Constants.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Operator.h"
#include "llvm/Support/Compiler.h"
@@ -412,6 +413,21 @@ bool ConstantRange::contains(const ConstantRange &Other) const {
return Other.getUpper().ule(Upper) && Lower.ule(Other.getLower());
}
+unsigned ConstantRange::getActiveBits() const {
+ if (isEmptySet())
+ return 0;
+
+ return getUnsignedMax().getActiveBits();
+}
+
+unsigned ConstantRange::getMinSignedBits() const {
+ if (isEmptySet())
+ return 0;
+
+ return std::max(getSignedMin().getMinSignedBits(),
+ getSignedMax().getMinSignedBits());
+}
+
ConstantRange ConstantRange::subtract(const APInt &Val) const {
assert(Val.getBitWidth() == getBitWidth() && "Wrong bit width");
// If the set is empty or full, don't modify the endpoints.
@@ -835,6 +851,54 @@ ConstantRange ConstantRange::overflowingBinaryOp(Instruction::BinaryOps BinOp,
}
}
+bool ConstantRange::isIntrinsicSupported(Intrinsic::ID IntrinsicID) {
+ switch (IntrinsicID) {
+ case Intrinsic::uadd_sat:
+ case Intrinsic::usub_sat:
+ case Intrinsic::sadd_sat:
+ case Intrinsic::ssub_sat:
+ case Intrinsic::umin:
+ case Intrinsic::umax:
+ case Intrinsic::smin:
+ case Intrinsic::smax:
+ case Intrinsic::abs:
+ return true;
+ default:
+ return false;
+ }
+}
+
+ConstantRange ConstantRange::intrinsic(Intrinsic::ID IntrinsicID,
+ ArrayRef<ConstantRange> Ops) {
+ switch (IntrinsicID) {
+ case Intrinsic::uadd_sat:
+ return Ops[0].uadd_sat(Ops[1]);
+ case Intrinsic::usub_sat:
+ return Ops[0].usub_sat(Ops[1]);
+ case Intrinsic::sadd_sat:
+ return Ops[0].sadd_sat(Ops[1]);
+ case Intrinsic::ssub_sat:
+ return Ops[0].ssub_sat(Ops[1]);
+ case Intrinsic::umin:
+ return Ops[0].umin(Ops[1]);
+ case Intrinsic::umax:
+ return Ops[0].umax(Ops[1]);
+ case Intrinsic::smin:
+ return Ops[0].smin(Ops[1]);
+ case Intrinsic::smax:
+ return Ops[0].smax(Ops[1]);
+ case Intrinsic::abs: {
+ const APInt *IntMinIsPoison = Ops[1].getSingleElement();
+ assert(IntMinIsPoison && "Must be known (immarg)");
+ assert(IntMinIsPoison->getBitWidth() == 1 && "Must be boolean");
+ return Ops[0].abs(IntMinIsPoison->getBoolValue());
+ }
+ default:
+ assert(!isIntrinsicSupported(IntrinsicID) && "Shouldn't be supported");
+ llvm_unreachable("Unsupported intrinsic");
+ }
+}
+
ConstantRange
ConstantRange::add(const ConstantRange &Other) const {
if (isEmptySet() || Other.isEmptySet())
@@ -1191,6 +1255,16 @@ ConstantRange ConstantRange::srem(const ConstantRange &RHS) const {
return ConstantRange(std::move(Lower), std::move(Upper));
}
+ConstantRange ConstantRange::binaryNot() const {
+ if (isEmptySet())
+ return getEmpty();
+
+ if (isWrappedSet())
+ return getFull();
+
+ return ConstantRange(APInt::getAllOnesValue(getBitWidth())).sub(*this);
+}
+
ConstantRange
ConstantRange::binaryAnd(const ConstantRange &Other) const {
if (isEmptySet() || Other.isEmptySet())
@@ -1229,6 +1303,12 @@ ConstantRange ConstantRange::binaryXor(const ConstantRange &Other) const {
if (isSingleElement() && Other.isSingleElement())
return {*getSingleElement() ^ *Other.getSingleElement()};
+ // Special-case binary complement, since we can give a precise answer.
+ if (Other.isSingleElement() && Other.getSingleElement()->isAllOnesValue())
+ return binaryNot();
+ if (isSingleElement() && getSingleElement()->isAllOnesValue())
+ return Other.binaryNot();
+
// TODO: replace this with something less conservative
return getFull();
}
@@ -1418,7 +1498,7 @@ ConstantRange ConstantRange::inverse() const {
return ConstantRange(Upper, Lower);
}
-ConstantRange ConstantRange::abs() const {
+ConstantRange ConstantRange::abs(bool IntMinIsPoison) const {
if (isEmptySet())
return getEmpty();
@@ -1430,12 +1510,23 @@ ConstantRange ConstantRange::abs() const {
else
Lo = APIntOps::umin(Lower, -Upper + 1);
- // SignedMin is included in the result range.
- return ConstantRange(Lo, APInt::getSignedMinValue(getBitWidth()) + 1);
+ // If SignedMin is not poison, then it is included in the result range.
+ if (IntMinIsPoison)
+ return ConstantRange(Lo, APInt::getSignedMinValue(getBitWidth()));
+ else
+ return ConstantRange(Lo, APInt::getSignedMinValue(getBitWidth()) + 1);
}
APInt SMin = getSignedMin(), SMax = getSignedMax();
+ // Skip SignedMin if it is poison.
+ if (IntMinIsPoison && SMin.isMinSignedValue()) {
+ // The range may become empty if it *only* contains SignedMin.
+ if (SMax.isMinSignedValue())
+ return getEmpty();
+ ++SMin;
+ }
+
// All non-negative.
if (SMin.isNonNegative())
return *this;
diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp
index cbbcca20ea51..6fd205c654a8 100644
--- a/llvm/lib/IR/Constants.cpp
+++ b/llvm/lib/IR/Constants.cpp
@@ -161,7 +161,7 @@ bool Constant::isNotOneValue() const {
// Check that vectors don't contain 1
if (auto *VTy = dyn_cast<VectorType>(this->getType())) {
- unsigned NumElts = VTy->getNumElements();
+ unsigned NumElts = cast<FixedVectorType>(VTy)->getNumElements();
for (unsigned i = 0; i != NumElts; ++i) {
Constant *Elt = this->getAggregateElement(i);
if (!Elt || !Elt->isNotOneValue())
@@ -211,7 +211,7 @@ bool Constant::isNotMinSignedValue() const {
// Check that vectors don't contain INT_MIN
if (auto *VTy = dyn_cast<VectorType>(this->getType())) {
- unsigned NumElts = VTy->getNumElements();
+ unsigned NumElts = cast<FixedVectorType>(VTy)->getNumElements();
for (unsigned i = 0; i != NumElts; ++i) {
Constant *Elt = this->getAggregateElement(i);
if (!Elt || !Elt->isNotMinSignedValue())
@@ -227,7 +227,7 @@ bool Constant::isNotMinSignedValue() const {
bool Constant::isFiniteNonZeroFP() const {
if (auto *CFP = dyn_cast<ConstantFP>(this))
return CFP->getValueAPF().isFiniteNonZero();
- auto *VTy = dyn_cast<VectorType>(getType());
+ auto *VTy = dyn_cast<FixedVectorType>(getType());
if (!VTy)
return false;
for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) {
@@ -304,23 +304,42 @@ bool Constant::isElementWiseEqual(Value *Y) const {
return isa<UndefValue>(CmpEq) || match(CmpEq, m_One());
}
-bool Constant::containsUndefElement() const {
- if (auto *VTy = dyn_cast<VectorType>(getType())) {
- for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
- if (isa<UndefValue>(getAggregateElement(i)))
+static bool
+containsUndefinedElement(const Constant *C,
+ function_ref<bool(const Constant *)> HasFn) {
+ if (auto *VTy = dyn_cast<VectorType>(C->getType())) {
+ if (HasFn(C))
+ return true;
+ if (isa<ConstantAggregateZero>(C))
+ return false;
+ if (isa<ScalableVectorType>(C->getType()))
+ return false;
+
+ for (unsigned i = 0, e = cast<FixedVectorType>(VTy)->getNumElements();
+ i != e; ++i)
+ if (HasFn(C->getAggregateElement(i)))
return true;
}
return false;
}
+bool Constant::containsUndefOrPoisonElement() const {
+ return containsUndefinedElement(
+ this, [&](const auto *C) { return isa<UndefValue>(C); });
+}
+
+bool Constant::containsPoisonElement() const {
+ return containsUndefinedElement(
+ this, [&](const auto *C) { return isa<PoisonValue>(C); });
+}
+
bool Constant::containsConstantExpression() const {
- if (auto *VTy = dyn_cast<VectorType>(getType())) {
+ if (auto *VTy = dyn_cast<FixedVectorType>(getType())) {
for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
if (isa<ConstantExpr>(getAggregateElement(i)))
return true;
}
-
return false;
}
@@ -400,16 +419,23 @@ Constant *Constant::getAllOnesValue(Type *Ty) {
}
Constant *Constant::getAggregateElement(unsigned Elt) const {
- if (const ConstantAggregate *CC = dyn_cast<ConstantAggregate>(this))
+ if (const auto *CC = dyn_cast<ConstantAggregate>(this))
return Elt < CC->getNumOperands() ? CC->getOperand(Elt) : nullptr;
- if (const ConstantAggregateZero *CAZ = dyn_cast<ConstantAggregateZero>(this))
+ // FIXME: getNumElements() will fail for non-fixed vector types.
+ if (isa<ScalableVectorType>(getType()))
+ return nullptr;
+
+ if (const auto *CAZ = dyn_cast<ConstantAggregateZero>(this))
return Elt < CAZ->getNumElements() ? CAZ->getElementValue(Elt) : nullptr;
- if (const UndefValue *UV = dyn_cast<UndefValue>(this))
+ if (const auto *PV = dyn_cast<PoisonValue>(this))
+ return Elt < PV->getNumElements() ? PV->getElementValue(Elt) : nullptr;
+
+ if (const auto *UV = dyn_cast<UndefValue>(this))
return Elt < UV->getNumElements() ? UV->getElementValue(Elt) : nullptr;
- if (const ConstantDataSequential *CDS =dyn_cast<ConstantDataSequential>(this))
+ if (const auto *CDS = dyn_cast<ConstantDataSequential>(this))
return Elt < CDS->getNumElements() ? CDS->getElementAsConstant(Elt)
: nullptr;
return nullptr;
@@ -501,9 +527,15 @@ void llvm::deleteConstant(Constant *C) {
case Constant::BlockAddressVal:
delete static_cast<BlockAddress *>(C);
break;
+ case Constant::DSOLocalEquivalentVal:
+ delete static_cast<DSOLocalEquivalent *>(C);
+ break;
case Constant::UndefValueVal:
delete static_cast<UndefValue *>(C);
break;
+ case Constant::PoisonValueVal:
+ delete static_cast<PoisonValue *>(C);
+ break;
case Constant::ConstantExprVal:
if (isa<UnaryConstantExpr>(C))
delete static_cast<UnaryConstantExpr *>(C);
@@ -646,10 +678,17 @@ bool Constant::needsRelocation() const {
return false;
// Relative pointers do not need to be dynamically relocated.
- if (auto *LHSGV = dyn_cast<GlobalValue>(LHSOp0->stripPointerCasts()))
- if (auto *RHSGV = dyn_cast<GlobalValue>(RHSOp0->stripPointerCasts()))
+ if (auto *RHSGV =
+ dyn_cast<GlobalValue>(RHSOp0->stripInBoundsConstantOffsets())) {
+ auto *LHS = LHSOp0->stripInBoundsConstantOffsets();
+ if (auto *LHSGV = dyn_cast<GlobalValue>(LHS)) {
if (LHSGV->isDSOLocal() && RHSGV->isDSOLocal())
return false;
+ } else if (isa<DSOLocalEquivalent>(LHS)) {
+ if (RHSGV->isDSOLocal())
+ return false;
+ }
+ }
}
}
}
@@ -729,6 +768,40 @@ Constant *Constant::replaceUndefsWith(Constant *C, Constant *Replacement) {
return ConstantVector::get(NewC);
}
+Constant *Constant::mergeUndefsWith(Constant *C, Constant *Other) {
+ assert(C && Other && "Expected non-nullptr constant arguments");
+ if (match(C, m_Undef()))
+ return C;
+
+ Type *Ty = C->getType();
+ if (match(Other, m_Undef()))
+ return UndefValue::get(Ty);
+
+ auto *VTy = dyn_cast<FixedVectorType>(Ty);
+ if (!VTy)
+ return C;
+
+ Type *EltTy = VTy->getElementType();
+ unsigned NumElts = VTy->getNumElements();
+ assert(isa<FixedVectorType>(Other->getType()) &&
+ cast<FixedVectorType>(Other->getType())->getNumElements() == NumElts &&
+ "Type mismatch");
+
+ bool FoundExtraUndef = false;
+ SmallVector<Constant *, 32> NewC(NumElts);
+ for (unsigned I = 0; I != NumElts; ++I) {
+ NewC[I] = C->getAggregateElement(I);
+ Constant *OtherEltC = Other->getAggregateElement(I);
+ assert(NewC[I] && OtherEltC && "Unknown vector element");
+ if (!match(NewC[I], m_Undef()) && match(OtherEltC, m_Undef())) {
+ NewC[I] = UndefValue::get(EltTy);
+ FoundExtraUndef = true;
+ }
+ }
+ if (FoundExtraUndef)
+ return ConstantVector::get(NewC);
+ return C;
+}
//===----------------------------------------------------------------------===//
// ConstantInt
@@ -753,6 +826,10 @@ ConstantInt *ConstantInt::getFalse(LLVMContext &Context) {
return pImpl->TheFalseVal;
}
+ConstantInt *ConstantInt::getBool(LLVMContext &Context, bool V) {
+ return V ? getTrue(Context) : getFalse(Context);
+}
+
Constant *ConstantInt::getTrue(Type *Ty) {
assert(Ty->isIntOrIntVectorTy(1) && "Type not i1 or vector of i1.");
ConstantInt *TrueC = ConstantInt::getTrue(Ty->getContext());
@@ -769,6 +846,10 @@ Constant *ConstantInt::getFalse(Type *Ty) {
return FalseC;
}
+Constant *ConstantInt::getBool(Type *Ty, bool V) {
+ return V ? getTrue(Ty) : getFalse(Ty);
+}
+
// Get a ConstantInt from an APInt.
ConstantInt *ConstantInt::get(LLVMContext &Context, const APInt &V) {
// get an existing value or the insertion position
@@ -830,30 +911,12 @@ void ConstantInt::destroyConstantImpl() {
// ConstantFP
//===----------------------------------------------------------------------===//
-static const fltSemantics *TypeToFloatSemantics(Type *Ty) {
- if (Ty->isHalfTy())
- return &APFloat::IEEEhalf();
- if (Ty->isBFloatTy())
- return &APFloat::BFloat();
- if (Ty->isFloatTy())
- return &APFloat::IEEEsingle();
- if (Ty->isDoubleTy())
- return &APFloat::IEEEdouble();
- if (Ty->isX86_FP80Ty())
- return &APFloat::x87DoubleExtended();
- else if (Ty->isFP128Ty())
- return &APFloat::IEEEquad();
-
- assert(Ty->isPPC_FP128Ty() && "Unknown FP format");
- return &APFloat::PPCDoubleDouble();
-}
-
Constant *ConstantFP::get(Type *Ty, double V) {
LLVMContext &Context = Ty->getContext();
APFloat FV(V);
bool ignored;
- FV.convert(*TypeToFloatSemantics(Ty->getScalarType()),
+ FV.convert(Ty->getScalarType()->getFltSemantics(),
APFloat::rmNearestTiesToEven, &ignored);
Constant *C = get(Context, FV);
@@ -879,7 +942,7 @@ Constant *ConstantFP::get(Type *Ty, const APFloat &V) {
Constant *ConstantFP::get(Type *Ty, StringRef Str) {
LLVMContext &Context = Ty->getContext();
- APFloat FV(*TypeToFloatSemantics(Ty->getScalarType()), Str);
+ APFloat FV(Ty->getScalarType()->getFltSemantics(), Str);
Constant *C = get(Context, FV);
// For vectors, broadcast the value.
@@ -890,7 +953,7 @@ Constant *ConstantFP::get(Type *Ty, StringRef Str) {
}
Constant *ConstantFP::getNaN(Type *Ty, bool Negative, uint64_t Payload) {
- const fltSemantics &Semantics = *TypeToFloatSemantics(Ty->getScalarType());
+ const fltSemantics &Semantics = Ty->getScalarType()->getFltSemantics();
APFloat NaN = APFloat::getNaN(Semantics, Negative, Payload);
Constant *C = get(Ty->getContext(), NaN);
@@ -901,7 +964,7 @@ Constant *ConstantFP::getNaN(Type *Ty, bool Negative, uint64_t Payload) {
}
Constant *ConstantFP::getQNaN(Type *Ty, bool Negative, APInt *Payload) {
- const fltSemantics &Semantics = *TypeToFloatSemantics(Ty->getScalarType());
+ const fltSemantics &Semantics = Ty->getScalarType()->getFltSemantics();
APFloat NaN = APFloat::getQNaN(Semantics, Negative, Payload);
Constant *C = get(Ty->getContext(), NaN);
@@ -912,7 +975,7 @@ Constant *ConstantFP::getQNaN(Type *Ty, bool Negative, APInt *Payload) {
}
Constant *ConstantFP::getSNaN(Type *Ty, bool Negative, APInt *Payload) {
- const fltSemantics &Semantics = *TypeToFloatSemantics(Ty->getScalarType());
+ const fltSemantics &Semantics = Ty->getScalarType()->getFltSemantics();
APFloat NaN = APFloat::getSNaN(Semantics, Negative, Payload);
Constant *C = get(Ty->getContext(), NaN);
@@ -923,7 +986,7 @@ Constant *ConstantFP::getSNaN(Type *Ty, bool Negative, APInt *Payload) {
}
Constant *ConstantFP::getNegativeZero(Type *Ty) {
- const fltSemantics &Semantics = *TypeToFloatSemantics(Ty->getScalarType());
+ const fltSemantics &Semantics = Ty->getScalarType()->getFltSemantics();
APFloat NegZero = APFloat::getZero(Semantics, /*Negative=*/true);
Constant *C = get(Ty->getContext(), NegZero);
@@ -949,24 +1012,7 @@ ConstantFP* ConstantFP::get(LLVMContext &Context, const APFloat& V) {
std::unique_ptr<ConstantFP> &Slot = pImpl->FPConstants[V];
if (!Slot) {
- Type *Ty;
- if (&V.getSemantics() == &APFloat::IEEEhalf())
- Ty = Type::getHalfTy(Context);
- else if (&V.getSemantics() == &APFloat::BFloat())
- Ty = Type::getBFloatTy(Context);
- else if (&V.getSemantics() == &APFloat::IEEEsingle())
- Ty = Type::getFloatTy(Context);
- else if (&V.getSemantics() == &APFloat::IEEEdouble())
- Ty = Type::getDoubleTy(Context);
- else if (&V.getSemantics() == &APFloat::x87DoubleExtended())
- Ty = Type::getX86_FP80Ty(Context);
- else if (&V.getSemantics() == &APFloat::IEEEquad())
- Ty = Type::getFP128Ty(Context);
- else {
- assert(&V.getSemantics() == &APFloat::PPCDoubleDouble() &&
- "Unknown FP format");
- Ty = Type::getPPC_FP128Ty(Context);
- }
+ Type *Ty = Type::getFloatingPointTy(Context, V.getSemantics());
Slot.reset(new ConstantFP(Ty, V));
}
@@ -974,7 +1020,7 @@ ConstantFP* ConstantFP::get(LLVMContext &Context, const APFloat& V) {
}
Constant *ConstantFP::getInfinity(Type *Ty, bool Negative) {
- const fltSemantics &Semantics = *TypeToFloatSemantics(Ty->getScalarType());
+ const fltSemantics &Semantics = Ty->getScalarType()->getFltSemantics();
Constant *C = get(Ty->getContext(), APFloat::getInf(Semantics, Negative));
if (VectorType *VTy = dyn_cast<VectorType>(Ty))
@@ -985,7 +1031,7 @@ Constant *ConstantFP::getInfinity(Type *Ty, bool Negative) {
ConstantFP::ConstantFP(Type *Ty, const APFloat &V)
: ConstantData(Ty, ConstantFPVal), Val(V) {
- assert(&V.getSemantics() == TypeToFloatSemantics(Ty) &&
+ assert(&V.getSemantics() == &Ty->getFltSemantics() &&
"FP type Mismatch");
}
@@ -1029,7 +1075,7 @@ unsigned ConstantAggregateZero::getNumElements() const {
if (auto *AT = dyn_cast<ArrayType>(Ty))
return AT->getNumElements();
if (auto *VT = dyn_cast<VectorType>(Ty))
- return VT->getNumElements();
+ return cast<FixedVectorType>(VT)->getNumElements();
return Ty->getStructNumElements();
}
@@ -1064,11 +1110,37 @@ unsigned UndefValue::getNumElements() const {
if (auto *AT = dyn_cast<ArrayType>(Ty))
return AT->getNumElements();
if (auto *VT = dyn_cast<VectorType>(Ty))
- return VT->getNumElements();
+ return cast<FixedVectorType>(VT)->getNumElements();
return Ty->getStructNumElements();
}
//===----------------------------------------------------------------------===//
+// PoisonValue Implementation
+//===----------------------------------------------------------------------===//
+
+PoisonValue *PoisonValue::getSequentialElement() const {
+ if (ArrayType *ATy = dyn_cast<ArrayType>(getType()))
+ return PoisonValue::get(ATy->getElementType());
+ return PoisonValue::get(cast<VectorType>(getType())->getElementType());
+}
+
+PoisonValue *PoisonValue::getStructElement(unsigned Elt) const {
+ return PoisonValue::get(getType()->getStructElementType(Elt));
+}
+
+PoisonValue *PoisonValue::getElementValue(Constant *C) const {
+ if (isa<ArrayType>(getType()) || isa<VectorType>(getType()))
+ return getSequentialElement();
+ return getStructElement(cast<ConstantInt>(C)->getZExtValue());
+}
+
+PoisonValue *PoisonValue::getElementValue(unsigned Idx) const {
+ if (isa<ArrayType>(getType()) || isa<VectorType>(getType()))
+ return getSequentialElement();
+ return getStructElement(Idx);
+}
+
+//===----------------------------------------------------------------------===//
// ConstantXXX Classes
//===----------------------------------------------------------------------===//
@@ -1246,7 +1318,7 @@ Constant *ConstantStruct::get(StructType *ST, ArrayRef<Constant*> V) {
ConstantVector::ConstantVector(VectorType *T, ArrayRef<Constant *> V)
: ConstantAggregate(T, ConstantVectorVal, V) {
- assert(V.size() == T->getNumElements() &&
+ assert(V.size() == cast<FixedVectorType>(T)->getNumElements() &&
"Invalid initializer for constant vector");
}
@@ -1267,17 +1339,20 @@ Constant *ConstantVector::getImpl(ArrayRef<Constant*> V) {
Constant *C = V[0];
bool isZero = C->isNullValue();
bool isUndef = isa<UndefValue>(C);
+ bool isPoison = isa<PoisonValue>(C);
if (isZero || isUndef) {
for (unsigned i = 1, e = V.size(); i != e; ++i)
if (V[i] != C) {
- isZero = isUndef = false;
+ isZero = isUndef = isPoison = false;
break;
}
}
if (isZero)
return ConstantAggregateZero::get(T);
+ if (isPoison)
+ return PoisonValue::get(T);
if (isUndef)
return UndefValue::get(T);
@@ -1292,14 +1367,14 @@ Constant *ConstantVector::getImpl(ArrayRef<Constant*> V) {
}
Constant *ConstantVector::getSplat(ElementCount EC, Constant *V) {
- if (!EC.Scalable) {
+ if (!EC.isScalable()) {
// If this splat is compatible with ConstantDataVector, use it instead of
// ConstantVector.
if ((isa<ConstantFP>(V) || isa<ConstantInt>(V)) &&
ConstantDataSequential::isElementTypeCompatible(V->getType()))
- return ConstantDataVector::getSplat(EC.Min, V);
+ return ConstantDataVector::getSplat(EC.getKnownMinValue(), V);
- SmallVector<Constant *, 32> Elts(EC.Min, V);
+ SmallVector<Constant *, 32> Elts(EC.getKnownMinValue(), V);
return get(Elts);
}
@@ -1316,7 +1391,7 @@ Constant *ConstantVector::getSplat(ElementCount EC, Constant *V) {
Constant *UndefV = UndefValue::get(VTy);
V = ConstantExpr::getInsertElement(UndefV, V, ConstantInt::get(I32Ty, 0));
// Build shuffle mask to perform the splat.
- SmallVector<int, 8> Zeros(EC.Min, 0);
+ SmallVector<int, 8> Zeros(EC.getKnownMinValue(), 0);
// Splat.
return ConstantExpr::getShuffleVector(V, UndefV, Zeros);
}
@@ -1441,6 +1516,8 @@ Constant *ConstantExpr::getWithOperands(ArrayRef<Constant *> Ops, Type *Ty,
OnlyIfReducedTy);
case Instruction::ExtractValue:
return ConstantExpr::getExtractValue(Ops[0], getIndices(), OnlyIfReducedTy);
+ case Instruction::FNeg:
+ return ConstantExpr::getFNeg(Ops[0]);
case Instruction::ShuffleVector:
return ConstantExpr::getShuffleVector(Ops[0], Ops[1], getShuffleMask(),
OnlyIfReducedTy);
@@ -1601,7 +1678,7 @@ Constant *Constant::getSplatValue(bool AllowUndefs) const {
ConstantInt *Index = dyn_cast<ConstantInt>(IElt->getOperand(2));
if (Index && Index->getValue() == 0 &&
- std::all_of(Mask.begin(), Mask.end(), [](int I) { return I == 0; }))
+ llvm::all_of(Mask, [](int I) { return I == 0; }))
return SplatVal;
}
}
@@ -1673,7 +1750,26 @@ UndefValue *UndefValue::get(Type *Ty) {
/// Remove the constant from the constant table.
void UndefValue::destroyConstantImpl() {
// Free the constant and any dangling references to it.
- getContext().pImpl->UVConstants.erase(getType());
+ if (getValueID() == UndefValueVal) {
+ getContext().pImpl->UVConstants.erase(getType());
+ } else if (getValueID() == PoisonValueVal) {
+ getContext().pImpl->PVConstants.erase(getType());
+ }
+ llvm_unreachable("Not a undef or a poison!");
+}
+
+PoisonValue *PoisonValue::get(Type *Ty) {
+ std::unique_ptr<PoisonValue> &Entry = Ty->getContext().pImpl->PVConstants[Ty];
+ if (!Entry)
+ Entry.reset(new PoisonValue(Ty));
+
+ return Entry.get();
+}
+
+/// Remove the constant from the constant table.
+void PoisonValue::destroyConstantImpl() {
+ // Free the constant and any dangling references to it.
+ getContext().pImpl->PVConstants.erase(getType());
}
BlockAddress *BlockAddress::get(BasicBlock *BB) {
@@ -1754,6 +1850,58 @@ Value *BlockAddress::handleOperandChangeImpl(Value *From, Value *To) {
return nullptr;
}
+DSOLocalEquivalent *DSOLocalEquivalent::get(GlobalValue *GV) {
+ DSOLocalEquivalent *&Equiv = GV->getContext().pImpl->DSOLocalEquivalents[GV];
+ if (!Equiv)
+ Equiv = new DSOLocalEquivalent(GV);
+
+ assert(Equiv->getGlobalValue() == GV &&
+ "DSOLocalFunction does not match the expected global value");
+ return Equiv;
+}
+
+DSOLocalEquivalent::DSOLocalEquivalent(GlobalValue *GV)
+ : Constant(GV->getType(), Value::DSOLocalEquivalentVal, &Op<0>(), 1) {
+ setOperand(0, GV);
+}
+
+/// Remove the constant from the constant table.
+void DSOLocalEquivalent::destroyConstantImpl() {
+ const GlobalValue *GV = getGlobalValue();
+ GV->getContext().pImpl->DSOLocalEquivalents.erase(GV);
+}
+
+Value *DSOLocalEquivalent::handleOperandChangeImpl(Value *From, Value *To) {
+ assert(From == getGlobalValue() && "Changing value does not match operand.");
+ assert(isa<Constant>(To) && "Can only replace the operands with a constant");
+
+ // The replacement is with another global value.
+ if (const auto *ToObj = dyn_cast<GlobalValue>(To)) {
+ DSOLocalEquivalent *&NewEquiv =
+ getContext().pImpl->DSOLocalEquivalents[ToObj];
+ if (NewEquiv)
+ return llvm::ConstantExpr::getBitCast(NewEquiv, getType());
+ }
+
+ // If the argument is replaced with a null value, just replace this constant
+ // with a null value.
+ if (cast<Constant>(To)->isNullValue())
+ return To;
+
+ // The replacement could be a bitcast or an alias to another function. We can
+ // replace it with a bitcast to the dso_local_equivalent of that function.
+ auto *Func = cast<Function>(To->stripPointerCastsAndAliases());
+ DSOLocalEquivalent *&NewEquiv = getContext().pImpl->DSOLocalEquivalents[Func];
+ if (NewEquiv)
+ return llvm::ConstantExpr::getBitCast(NewEquiv, getType());
+
+ // Replace this with the new one.
+ getContext().pImpl->DSOLocalEquivalents.erase(getGlobalValue());
+ NewEquiv = this;
+ setOperand(0, Func);
+ return nullptr;
+}
+
//---- ConstantExpr::get() implementations.
//
@@ -2002,8 +2150,8 @@ Constant *ConstantExpr::getPtrToInt(Constant *C, Type *DstTy,
"PtrToInt destination must be integer or integer vector");
assert(isa<VectorType>(C->getType()) == isa<VectorType>(DstTy));
if (isa<VectorType>(C->getType()))
- assert(cast<VectorType>(C->getType())->getNumElements() ==
- cast<VectorType>(DstTy)->getNumElements() &&
+ assert(cast<FixedVectorType>(C->getType())->getNumElements() ==
+ cast<FixedVectorType>(DstTy)->getNumElements() &&
"Invalid cast between a different number of vector elements");
return getFoldedCast(Instruction::PtrToInt, C, DstTy, OnlyIfReduced);
}
@@ -2016,8 +2164,8 @@ Constant *ConstantExpr::getIntToPtr(Constant *C, Type *DstTy,
"IntToPtr destination must be a pointer or pointer vector");
assert(isa<VectorType>(C->getType()) == isa<VectorType>(DstTy));
if (isa<VectorType>(C->getType()))
- assert(cast<VectorType>(C->getType())->getNumElements() ==
- cast<VectorType>(DstTy)->getNumElements() &&
+ assert(cast<VectorType>(C->getType())->getElementCount() ==
+ cast<VectorType>(DstTy)->getElementCount() &&
"Invalid cast between a different number of vector elements");
return getFoldedCast(Instruction::IntToPtr, C, DstTy, OnlyIfReduced);
}
@@ -2048,7 +2196,8 @@ Constant *ConstantExpr::getAddrSpaceCast(Constant *C, Type *DstTy,
Type *MidTy = PointerType::get(DstElemTy, SrcScalarTy->getAddressSpace());
if (VectorType *VT = dyn_cast<VectorType>(DstTy)) {
// Handle vectors of pointers.
- MidTy = FixedVectorType::get(MidTy, VT->getNumElements());
+ MidTy = FixedVectorType::get(MidTy,
+ cast<FixedVectorType>(VT)->getNumElements());
}
C = getBitCast(C, MidTy);
}
@@ -2245,7 +2394,7 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C,
unsigned AS = C->getType()->getPointerAddressSpace();
Type *ReqTy = DestTy->getPointerTo(AS);
- ElementCount EltCount = {0, false};
+ auto EltCount = ElementCount::getFixed(0);
if (VectorType *VecTy = dyn_cast<VectorType>(C->getType()))
EltCount = VecTy->getElementCount();
else
@@ -2253,7 +2402,7 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C,
if (VectorType *VecTy = dyn_cast<VectorType>(Idx->getType()))
EltCount = VecTy->getElementCount();
- if (EltCount.Min != 0)
+ if (EltCount.isNonZero())
ReqTy = VectorType::get(ReqTy, EltCount);
if (OnlyIfReducedTy == ReqTy)
@@ -2273,7 +2422,7 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C,
if (GTI.isStruct() && Idx->getType()->isVectorTy()) {
Idx = Idx->getSplatValue();
- } else if (GTI.isSequential() && EltCount.Min != 0 &&
+ } else if (GTI.isSequential() && EltCount.isNonZero() &&
!Idx->getType()->isVectorTy()) {
Idx = ConstantVector::getSplat(EltCount, Idx);
}
@@ -2549,6 +2698,11 @@ Constant *ConstantExpr::getXor(Constant *C1, Constant *C2) {
return get(Instruction::Xor, C1, C2);
}
+Constant *ConstantExpr::getUMin(Constant *C1, Constant *C2) {
+ Constant *Cmp = ConstantExpr::getICmp(CmpInst::ICMP_ULT, C1, C2);
+ return getSelect(Cmp, C1, C2);
+}
+
Constant *ConstantExpr::getShl(Constant *C1, Constant *C2,
bool HasNUW, bool HasNSW) {
unsigned Flags = (HasNUW ? OverflowingBinaryOperator::NoUnsignedWrap : 0) |
@@ -2566,6 +2720,35 @@ Constant *ConstantExpr::getAShr(Constant *C1, Constant *C2, bool isExact) {
isExact ? PossiblyExactOperator::IsExact : 0);
}
+Constant *ConstantExpr::getExactLogBase2(Constant *C) {
+ Type *Ty = C->getType();
+ const APInt *IVal;
+ if (match(C, m_APInt(IVal)) && IVal->isPowerOf2())
+ return ConstantInt::get(Ty, IVal->logBase2());
+
+ // FIXME: We can extract pow of 2 of splat constant for scalable vectors.
+ auto *VecTy = dyn_cast<FixedVectorType>(Ty);
+ if (!VecTy)
+ return nullptr;
+
+ SmallVector<Constant *, 4> Elts;
+ for (unsigned I = 0, E = VecTy->getNumElements(); I != E; ++I) {
+ Constant *Elt = C->getAggregateElement(I);
+ if (!Elt)
+ return nullptr;
+ // Note that log2(iN undef) is *NOT* iN undef, because log2(iN undef) u< N.
+ if (isa<UndefValue>(Elt)) {
+ Elts.push_back(Constant::getNullValue(Ty->getScalarType()));
+ continue;
+ }
+ if (!match(Elt, m_APInt(IVal)) || !IVal->isPowerOf2())
+ return nullptr;
+ Elts.push_back(ConstantInt::get(Ty->getScalarType(), IVal->logBase2()));
+ }
+
+ return ConstantVector::get(Elts);
+}
+
Constant *ConstantExpr::getBinOpIdentity(unsigned Opcode, Type *Ty,
bool AllowRHSConstant) {
assert(Instruction::isBinaryOp(Opcode) && "Only binops allowed");
@@ -2690,7 +2873,7 @@ bool ConstantDataSequential::isElementTypeCompatible(Type *Ty) {
unsigned ConstantDataSequential::getNumElements() const {
if (ArrayType *AT = dyn_cast<ArrayType>(getType()))
return AT->getNumElements();
- return cast<VectorType>(getType())->getNumElements();
+ return cast<FixedVectorType>(getType())->getNumElements();
}
@@ -2739,56 +2922,58 @@ Constant *ConstantDataSequential::getImpl(StringRef Elements, Type *Ty) {
// body but different types. For example, 0,0,0,1 could be a 4 element array
// of i8, or a 1-element array of i32. They'll both end up in the same
/// StringMap bucket, linked up by their Next pointers. Walk the list.
- ConstantDataSequential **Entry = &Slot.second;
- for (ConstantDataSequential *Node = *Entry; Node;
- Entry = &Node->Next, Node = *Entry)
- if (Node->getType() == Ty)
- return Node;
+ std::unique_ptr<ConstantDataSequential> *Entry = &Slot.second;
+ for (; *Entry; Entry = &(*Entry)->Next)
+ if ((*Entry)->getType() == Ty)
+ return Entry->get();
// Okay, we didn't get a hit. Create a node of the right class, link it in,
// and return it.
- if (isa<ArrayType>(Ty))
- return *Entry = new ConstantDataArray(Ty, Slot.first().data());
+ if (isa<ArrayType>(Ty)) {
+ // Use reset because std::make_unique can't access the constructor.
+ Entry->reset(new ConstantDataArray(Ty, Slot.first().data()));
+ return Entry->get();
+ }
assert(isa<VectorType>(Ty));
- return *Entry = new ConstantDataVector(Ty, Slot.first().data());
+ // Use reset because std::make_unique can't access the constructor.
+ Entry->reset(new ConstantDataVector(Ty, Slot.first().data()));
+ return Entry->get();
}
void ConstantDataSequential::destroyConstantImpl() {
// Remove the constant from the StringMap.
- StringMap<ConstantDataSequential*> &CDSConstants =
- getType()->getContext().pImpl->CDSConstants;
+ StringMap<std::unique_ptr<ConstantDataSequential>> &CDSConstants =
+ getType()->getContext().pImpl->CDSConstants;
- StringMap<ConstantDataSequential*>::iterator Slot =
- CDSConstants.find(getRawDataValues());
+ auto Slot = CDSConstants.find(getRawDataValues());
assert(Slot != CDSConstants.end() && "CDS not found in uniquing table");
- ConstantDataSequential **Entry = &Slot->getValue();
+ std::unique_ptr<ConstantDataSequential> *Entry = &Slot->getValue();
// Remove the entry from the hash table.
if (!(*Entry)->Next) {
// If there is only one value in the bucket (common case) it must be this
// entry, and removing the entry should remove the bucket completely.
- assert((*Entry) == this && "Hash mismatch in ConstantDataSequential");
+ assert(Entry->get() == this && "Hash mismatch in ConstantDataSequential");
getContext().pImpl->CDSConstants.erase(Slot);
- } else {
- // Otherwise, there are multiple entries linked off the bucket, unlink the
- // node we care about but keep the bucket around.
- for (ConstantDataSequential *Node = *Entry; ;
- Entry = &Node->Next, Node = *Entry) {
- assert(Node && "Didn't find entry in its uniquing hash table!");
- // If we found our entry, unlink it from the list and we're done.
- if (Node == this) {
- *Entry = Node->Next;
- break;
- }
- }
+ return;
}
- // If we were part of a list, make sure that we don't delete the list that is
- // still owned by the uniquing map.
- Next = nullptr;
+ // Otherwise, there are multiple entries linked off the bucket, unlink the
+ // node we care about but keep the bucket around.
+ while (true) {
+ std::unique_ptr<ConstantDataSequential> &Node = *Entry;
+ assert(Node && "Didn't find entry in its uniquing hash table!");
+ // If we found our entry, unlink it from the list and we're done.
+ if (Node.get() == this) {
+ Node = std::move(Node->Next);
+ return;
+ }
+
+ Entry = &Node->Next;
+ }
}
/// getFP() constructors - Return a constant of array type with a float
@@ -2938,7 +3123,7 @@ Constant *ConstantDataVector::getSplat(unsigned NumElts, Constant *V) {
return getFP(V->getType(), Elts);
}
}
- return ConstantVector::getSplat({NumElts, false}, V);
+ return ConstantVector::getSplat(ElementCount::getFixed(NumElts), V);
}
@@ -3248,7 +3433,7 @@ Value *ConstantExpr::handleOperandChangeImpl(Value *From, Value *ToV) {
}
Instruction *ConstantExpr::getAsInstruction() const {
- SmallVector<Value *, 4> ValueOperands(op_begin(), op_end());
+ SmallVector<Value *, 4> ValueOperands(operands());
ArrayRef<Value*> Ops(ValueOperands);
switch (getOpcode()) {
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index 6f3bbc80d4fd..90ba69069bae 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -12,7 +12,6 @@
//===----------------------------------------------------------------------===//
#include "llvm-c/Core.h"
-#include "llvm/ADT/StringSwitch.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugInfoMetadata.h"
@@ -146,6 +145,11 @@ LLVMAttributeRef LLVMCreateEnumAttribute(LLVMContextRef C, unsigned KindID,
return wrap(Attribute::getWithByValType(Ctx, NULL));
}
+ if (AttrKind == Attribute::AttrKind::StructRet) {
+ // Same as byval.
+ return wrap(Attribute::getWithStructRetType(Ctx, NULL));
+ }
+
return wrap(Attribute::get(Ctx, AttrKind, Val));
}
@@ -507,6 +511,8 @@ LLVMTypeKind LLVMGetTypeKind(LLVMTypeRef Ty) {
return LLVMVectorTypeKind;
case Type::X86_MMXTyID:
return LLVMX86_MMXTypeKind;
+ case Type::X86_AMXTyID:
+ return LLVMX86_AMXTypeKind;
case Type::TokenTyID:
return LLVMTokenTypeKind;
case Type::ScalableVectorTyID:
@@ -618,6 +624,9 @@ LLVMTypeRef LLVMPPCFP128TypeInContext(LLVMContextRef C) {
LLVMTypeRef LLVMX86MMXTypeInContext(LLVMContextRef C) {
return (LLVMTypeRef) Type::getX86_MMXTy(*unwrap(C));
}
+LLVMTypeRef LLVMX86AMXTypeInContext(LLVMContextRef C) {
+ return (LLVMTypeRef) Type::getX86_AMXTy(*unwrap(C));
+}
LLVMTypeRef LLVMHalfType(void) {
return LLVMHalfTypeInContext(LLVMGetGlobalContext());
@@ -643,6 +652,9 @@ LLVMTypeRef LLVMPPCFP128Type(void) {
LLVMTypeRef LLVMX86MMXType(void) {
return LLVMX86MMXTypeInContext(LLVMGetGlobalContext());
}
+LLVMTypeRef LLVMX86AMXType(void) {
+ return LLVMX86AMXTypeInContext(LLVMGetGlobalContext());
+}
/*--.. Operations on function types ........................................--*/
@@ -734,7 +746,11 @@ LLVMBool LLVMIsLiteralStruct(LLVMTypeRef StructTy) {
}
LLVMTypeRef LLVMGetTypeByName(LLVMModuleRef M, const char *Name) {
- return wrap(unwrap(M)->getTypeByName(Name));
+ return wrap(StructType::getTypeByName(unwrap(M)->getContext(), Name));
+}
+
+LLVMTypeRef LLVMGetTypeByName2(LLVMContextRef C, const char *Name) {
+ return wrap(StructType::getTypeByName(*unwrap(C), Name));
}
/*--.. Operations on array, pointer, and vector types (sequence types) .....--*/
@@ -759,6 +775,11 @@ LLVMTypeRef LLVMVectorType(LLVMTypeRef ElementType, unsigned ElementCount) {
return wrap(FixedVectorType::get(unwrap(ElementType), ElementCount));
}
+LLVMTypeRef LLVMScalableVectorType(LLVMTypeRef ElementType,
+ unsigned ElementCount) {
+ return wrap(ScalableVectorType::get(unwrap(ElementType), ElementCount));
+}
+
LLVMTypeRef LLVMGetElementType(LLVMTypeRef WrappedTy) {
auto *Ty = unwrap<Type>(WrappedTy);
if (auto *PTy = dyn_cast<PointerType>(Ty))
@@ -781,7 +802,7 @@ unsigned LLVMGetPointerAddressSpace(LLVMTypeRef PointerTy) {
}
unsigned LLVMGetVectorSize(LLVMTypeRef VectorTy) {
- return unwrap<VectorType>(VectorTy)->getNumElements();
+ return unwrap<VectorType>(VectorTy)->getElementCount().getKnownMinValue();
}
/*--.. Operations on other types ...........................................--*/
@@ -816,6 +837,7 @@ LLVMTypeRef LLVMTypeOf(LLVMValueRef Val) {
LLVMValueKind LLVMGetValueKind(LLVMValueRef Val) {
switch(unwrap(Val)->getValueID()) {
+#define LLVM_C_API 1
#define HANDLE_VALUE(Name) \
case Value::Name##Val: \
return LLVM##Name##ValueKind;
@@ -925,6 +947,7 @@ LLVMValueMetadataEntry *
LLVMInstructionGetAllMetadataOtherThanDebugLoc(LLVMValueRef Value,
size_t *NumEntries) {
return llvm_getMetadata(NumEntries, [&Value](MetadataEntries &Entries) {
+ Entries.clear();
unwrap<Instruction>(Value)->getAllMetadata(Entries);
});
}
@@ -1034,6 +1057,10 @@ LLVMValueRef LLVMGetUndef(LLVMTypeRef Ty) {
return wrap(UndefValue::get(unwrap(Ty)));
}
+LLVMValueRef LLVMGetPoison(LLVMTypeRef Ty) {
+ return wrap(PoisonValue::get(unwrap(Ty)));
+}
+
LLVMBool LLVMIsConstant(LLVMValueRef Ty) {
return isa<Constant>(unwrap(Ty));
}
@@ -1048,6 +1075,10 @@ LLVMBool LLVMIsUndef(LLVMValueRef Val) {
return isa<UndefValue>(unwrap(Val));
}
+LLVMBool LLVMIsPoison(LLVMValueRef Val) {
+ return isa<PoisonValue>(unwrap(Val));
+}
+
LLVMValueRef LLVMConstPointerNull(LLVMTypeRef Ty) {
return wrap(ConstantPointerNull::get(unwrap<PointerType>(Ty)));
}
@@ -2034,6 +2065,7 @@ void LLVMSetAlignment(LLVMValueRef V, unsigned Bytes) {
LLVMValueMetadataEntry *LLVMGlobalCopyAllMetadata(LLVMValueRef Value,
size_t *NumEntries) {
return llvm_getMetadata(NumEntries, [&Value](MetadataEntries &Entries) {
+ Entries.clear();
if (Instruction *Instr = dyn_cast<Instruction>(unwrap(Value))) {
Instr->getAllMetadata(Entries);
} else {
@@ -3952,6 +3984,19 @@ LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef Ptr,
singleThread ? SyncScope::SingleThread : SyncScope::System));
}
+unsigned LLVMGetNumMaskElements(LLVMValueRef SVInst) {
+ Value *P = unwrap<Value>(SVInst);
+ ShuffleVectorInst *I = cast<ShuffleVectorInst>(P);
+ return I->getShuffleMask().size();
+}
+
+int LLVMGetMaskValue(LLVMValueRef SVInst, unsigned Elt) {
+ Value *P = unwrap<Value>(SVInst);
+ ShuffleVectorInst *I = cast<ShuffleVectorInst>(P);
+ return I->getMaskValue(Elt);
+}
+
+int LLVMGetUndefMaskElem(void) { return UndefMaskElem; }
LLVMBool LLVMIsAtomicSingleThread(LLVMValueRef AtomicInst) {
Value *P = unwrap<Value>(AtomicInst);
diff --git a/llvm/lib/IR/DIBuilder.cpp b/llvm/lib/IR/DIBuilder.cpp
index 45cbbb3a6037..5104dc349d0b 100644
--- a/llvm/lib/IR/DIBuilder.cpp
+++ b/llvm/lib/IR/DIBuilder.cpp
@@ -267,6 +267,12 @@ DIBasicType *DIBuilder::createBasicType(StringRef Name, uint64_t SizeInBits,
0, Encoding, Flags);
}
+DIStringType *DIBuilder::createStringType(StringRef Name, uint64_t SizeInBits) {
+ assert(!Name.empty() && "Unable to create type without name");
+ return DIStringType::get(VMContext, dwarf::DW_TAG_string_type, Name,
+ SizeInBits, 0);
+}
+
DIDerivedType *DIBuilder::createQualifiedType(unsigned Tag, DIType *FromTy) {
return DIDerivedType::get(VMContext, Tag, "", nullptr, 0, nullptr, FromTy, 0,
0, 0, None, DINode::FlagZero);
@@ -519,12 +525,24 @@ DICompositeType *DIBuilder::createEnumerationType(
return CTy;
}
-DICompositeType *DIBuilder::createArrayType(uint64_t Size,
- uint32_t AlignInBits, DIType *Ty,
- DINodeArray Subscripts) {
- auto *R = DICompositeType::get(VMContext, dwarf::DW_TAG_array_type, "",
- nullptr, 0, nullptr, Ty, Size, AlignInBits, 0,
- DINode::FlagZero, Subscripts, 0, nullptr);
+DICompositeType *DIBuilder::createArrayType(
+ uint64_t Size, uint32_t AlignInBits, DIType *Ty, DINodeArray Subscripts,
+ PointerUnion<DIExpression *, DIVariable *> DL,
+ PointerUnion<DIExpression *, DIVariable *> AS,
+ PointerUnion<DIExpression *, DIVariable *> AL,
+ PointerUnion<DIExpression *, DIVariable *> RK) {
+ auto *R = DICompositeType::get(
+ VMContext, dwarf::DW_TAG_array_type, "", nullptr, 0,
+ nullptr, Ty, Size, AlignInBits, 0, DINode::FlagZero,
+ Subscripts, 0, nullptr, nullptr, "", nullptr,
+ DL.is<DIExpression *>() ? (Metadata *)DL.get<DIExpression *>()
+ : (Metadata *)DL.get<DIVariable *>(),
+ AS.is<DIExpression *>() ? (Metadata *)AS.get<DIExpression *>()
+ : (Metadata *)AS.get<DIVariable *>(),
+ AL.is<DIExpression *>() ? (Metadata *)AL.get<DIExpression *>()
+ : (Metadata *)AL.get<DIVariable *>(),
+ RK.is<DIExpression *>() ? (Metadata *)RK.get<DIExpression *>()
+ : (Metadata *)RK.get<DIVariable *>());
trackIfUnresolved(R);
return R;
}
@@ -643,6 +661,18 @@ DISubrange *DIBuilder::getOrCreateSubrange(Metadata *CountNode, Metadata *LB,
return DISubrange::get(VMContext, CountNode, LB, UB, Stride);
}
+DIGenericSubrange *DIBuilder::getOrCreateGenericSubrange(
+ DIGenericSubrange::BoundType CountNode, DIGenericSubrange::BoundType LB,
+ DIGenericSubrange::BoundType UB, DIGenericSubrange::BoundType Stride) {
+ auto ConvToMetadata = [&](DIGenericSubrange::BoundType Bound) -> Metadata * {
+ return Bound.is<DIExpression *>() ? (Metadata *)Bound.get<DIExpression *>()
+ : (Metadata *)Bound.get<DIVariable *>();
+ };
+ return DIGenericSubrange::get(VMContext, ConvToMetadata(CountNode),
+ ConvToMetadata(LB), ConvToMetadata(UB),
+ ConvToMetadata(Stride));
+}
+
static void checkGlobalVariableScope(DIScope *Context) {
#ifndef NDEBUG
if (auto *CT =
@@ -844,9 +874,10 @@ DINamespace *DIBuilder::createNameSpace(DIScope *Scope, StringRef Name,
DIModule *DIBuilder::createModule(DIScope *Scope, StringRef Name,
StringRef ConfigurationMacros,
StringRef IncludePath, StringRef APINotesFile,
- DIFile *File, unsigned LineNo) {
+ DIFile *File, unsigned LineNo, bool IsDecl) {
return DIModule::get(VMContext, File, getNonCompileUnitScope(Scope), Name,
- ConfigurationMacros, IncludePath, APINotesFile, LineNo);
+ ConfigurationMacros, IncludePath, APINotesFile, LineNo,
+ IsDecl);
}
DILexicalBlockFile *DIBuilder::createLexicalBlockFile(DIScope *Scope,
diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp
index c44737c5bfc2..274ea0aa5fd1 100644
--- a/llvm/lib/IR/DataLayout.cpp
+++ b/llvm/lib/IR/DataLayout.cpp
@@ -27,6 +27,7 @@
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Casting.h"
+#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/TypeSize.h"
@@ -64,7 +65,8 @@ StructLayout::StructLayout(StructType *ST, const DataLayout &DL) {
StructAlignment = std::max(TyAlign, StructAlignment);
MemberOffsets[i] = StructSize;
- StructSize += DL.getTypeAllocSize(Ty); // Consume space for this data item
+ // Consume space for this data item
+ StructSize += DL.getTypeAllocSize(Ty).getFixedValue();
}
// Add padding to the end of the struct so that it could be put in an array
@@ -181,6 +183,7 @@ void DataLayout::reset(StringRef Desc) {
AllocaAddrSpace = 0;
StackNaturalAlign.reset();
ProgramAddrSpace = 0;
+ DefaultGlobalsAddrSpace = 0;
FunctionPtrAlign.reset();
TheFunctionPtrAlignType = FunctionPtrAlignType::Independent;
ManglingMode = MM_None;
@@ -188,57 +191,80 @@ void DataLayout::reset(StringRef Desc) {
// Default alignments
for (const LayoutAlignElem &E : DefaultAlignments) {
- setAlignment((AlignTypeEnum)E.AlignType, E.ABIAlign, E.PrefAlign,
- E.TypeBitWidth);
+ if (Error Err = setAlignment((AlignTypeEnum)E.AlignType, E.ABIAlign,
+ E.PrefAlign, E.TypeBitWidth))
+ return report_fatal_error(std::move(Err));
}
- setPointerAlignment(0, Align(8), Align(8), 8, 8);
+ if (Error Err = setPointerAlignment(0, Align(8), Align(8), 8, 8))
+ return report_fatal_error(std::move(Err));
- parseSpecifier(Desc);
+ if (Error Err = parseSpecifier(Desc))
+ return report_fatal_error(std::move(Err));
+}
+
+Expected<DataLayout> DataLayout::parse(StringRef LayoutDescription) {
+ DataLayout Layout("");
+ if (Error Err = Layout.parseSpecifier(LayoutDescription))
+ return std::move(Err);
+ return Layout;
+}
+
+static Error reportError(const Twine &Message) {
+ return createStringError(inconvertibleErrorCode(), Message);
}
/// Checked version of split, to ensure mandatory subparts.
-static std::pair<StringRef, StringRef> split(StringRef Str, char Separator) {
+static Error split(StringRef Str, char Separator,
+ std::pair<StringRef, StringRef> &Split) {
assert(!Str.empty() && "parse error, string can't be empty here");
- std::pair<StringRef, StringRef> Split = Str.split(Separator);
+ Split = Str.split(Separator);
if (Split.second.empty() && Split.first != Str)
- report_fatal_error("Trailing separator in datalayout string");
+ return reportError("Trailing separator in datalayout string");
if (!Split.second.empty() && Split.first.empty())
- report_fatal_error("Expected token before separator in datalayout string");
- return Split;
+ return reportError("Expected token before separator in datalayout string");
+ return Error::success();
}
/// Get an unsigned integer, including error checks.
-static unsigned getInt(StringRef R) {
- unsigned Result;
+template <typename IntTy> static Error getInt(StringRef R, IntTy &Result) {
bool error = R.getAsInteger(10, Result); (void)error;
if (error)
- report_fatal_error("not a number, or does not fit in an unsigned int");
- return Result;
+ return reportError("not a number, or does not fit in an unsigned int");
+ return Error::success();
}
-/// Convert bits into bytes. Assert if not a byte width multiple.
-static unsigned inBytes(unsigned Bits) {
- if (Bits % 8)
- report_fatal_error("number of bits must be a byte width multiple");
- return Bits / 8;
+/// Get an unsigned integer representing the number of bits and convert it into
+/// bytes. Error out of not a byte width multiple.
+template <typename IntTy>
+static Error getIntInBytes(StringRef R, IntTy &Result) {
+ if (Error Err = getInt<IntTy>(R, Result))
+ return Err;
+ if (Result % 8)
+ return reportError("number of bits must be a byte width multiple");
+ Result /= 8;
+ return Error::success();
}
-static unsigned getAddrSpace(StringRef R) {
- unsigned AddrSpace = getInt(R);
+static Error getAddrSpace(StringRef R, unsigned &AddrSpace) {
+ if (Error Err = getInt(R, AddrSpace))
+ return Err;
if (!isUInt<24>(AddrSpace))
- report_fatal_error("Invalid address space, must be a 24-bit integer");
- return AddrSpace;
+ return reportError("Invalid address space, must be a 24-bit integer");
+ return Error::success();
}
-void DataLayout::parseSpecifier(StringRef Desc) {
+Error DataLayout::parseSpecifier(StringRef Desc) {
StringRepresentation = std::string(Desc);
while (!Desc.empty()) {
// Split at '-'.
- std::pair<StringRef, StringRef> Split = split(Desc, '-');
+ std::pair<StringRef, StringRef> Split;
+ if (Error Err = split(Desc, '-', Split))
+ return Err;
Desc = Split.second;
// Split at ':'.
- Split = split(Split.first, ':');
+ if (Error Err = split(Split.first, ':', Split))
+ return Err;
// Aliases used below.
StringRef &Tok = Split.first; // Current token.
@@ -246,11 +272,14 @@ void DataLayout::parseSpecifier(StringRef Desc) {
if (Tok == "ni") {
do {
- Split = split(Rest, ':');
+ if (Error Err = split(Rest, ':', Split))
+ return Err;
Rest = Split.second;
- unsigned AS = getInt(Split.first);
+ unsigned AS;
+ if (Error Err = getInt(Split.first, AS))
+ return Err;
if (AS == 0)
- report_fatal_error("Address space 0 can never be non-integral");
+ return reportError("Address space 0 can never be non-integral");
NonIntegralAddressSpaces.push_back(AS);
} while (!Rest.empty());
@@ -273,28 +302,36 @@ void DataLayout::parseSpecifier(StringRef Desc) {
break;
case 'p': {
// Address space.
- unsigned AddrSpace = Tok.empty() ? 0 : getInt(Tok);
+ unsigned AddrSpace = 0;
+ if (!Tok.empty())
+ if (Error Err = getInt(Tok, AddrSpace))
+ return Err;
if (!isUInt<24>(AddrSpace))
- report_fatal_error("Invalid address space, must be a 24bit integer");
+ return reportError("Invalid address space, must be a 24bit integer");
// Size.
if (Rest.empty())
- report_fatal_error(
+ return reportError(
"Missing size specification for pointer in datalayout string");
- Split = split(Rest, ':');
- unsigned PointerMemSize = inBytes(getInt(Tok));
+ if (Error Err = split(Rest, ':', Split))
+ return Err;
+ unsigned PointerMemSize;
+ if (Error Err = getIntInBytes(Tok, PointerMemSize))
+ return Err;
if (!PointerMemSize)
- report_fatal_error("Invalid pointer size of 0 bytes");
+ return reportError("Invalid pointer size of 0 bytes");
// ABI alignment.
if (Rest.empty())
- report_fatal_error(
+ return reportError(
"Missing alignment specification for pointer in datalayout string");
- Split = split(Rest, ':');
- unsigned PointerABIAlign = inBytes(getInt(Tok));
+ if (Error Err = split(Rest, ':', Split))
+ return Err;
+ unsigned PointerABIAlign;
+ if (Error Err = getIntInBytes(Tok, PointerABIAlign))
+ return Err;
if (!isPowerOf2_64(PointerABIAlign))
- report_fatal_error(
- "Pointer ABI alignment must be a power of 2");
+ return reportError("Pointer ABI alignment must be a power of 2");
// Size of index used in GEP for address calculation.
// The parameter is optional. By default it is equal to size of pointer.
@@ -303,23 +340,28 @@ void DataLayout::parseSpecifier(StringRef Desc) {
// Preferred alignment.
unsigned PointerPrefAlign = PointerABIAlign;
if (!Rest.empty()) {
- Split = split(Rest, ':');
- PointerPrefAlign = inBytes(getInt(Tok));
+ if (Error Err = split(Rest, ':', Split))
+ return Err;
+ if (Error Err = getIntInBytes(Tok, PointerPrefAlign))
+ return Err;
if (!isPowerOf2_64(PointerPrefAlign))
- report_fatal_error(
- "Pointer preferred alignment must be a power of 2");
+ return reportError(
+ "Pointer preferred alignment must be a power of 2");
// Now read the index. It is the second optional parameter here.
if (!Rest.empty()) {
- Split = split(Rest, ':');
- IndexSize = inBytes(getInt(Tok));
+ if (Error Err = split(Rest, ':', Split))
+ return Err;
+ if (Error Err = getIntInBytes(Tok, IndexSize))
+ return Err;
if (!IndexSize)
- report_fatal_error("Invalid index size of 0 bytes");
+ return reportError("Invalid index size of 0 bytes");
}
}
- setPointerAlignment(AddrSpace, assumeAligned(PointerABIAlign),
- assumeAligned(PointerPrefAlign), PointerMemSize,
- IndexSize);
+ if (Error Err = setPointerAlignment(
+ AddrSpace, assumeAligned(PointerABIAlign),
+ assumeAligned(PointerPrefAlign), PointerMemSize, IndexSize))
+ return Err;
break;
}
case 'i':
@@ -336,61 +378,75 @@ void DataLayout::parseSpecifier(StringRef Desc) {
}
// Bit size.
- unsigned Size = Tok.empty() ? 0 : getInt(Tok);
+ unsigned Size = 0;
+ if (!Tok.empty())
+ if (Error Err = getInt(Tok, Size))
+ return Err;
if (AlignType == AGGREGATE_ALIGN && Size != 0)
- report_fatal_error(
+ return reportError(
"Sized aggregate specification in datalayout string");
// ABI alignment.
if (Rest.empty())
- report_fatal_error(
+ return reportError(
"Missing alignment specification in datalayout string");
- Split = split(Rest, ':');
- const unsigned ABIAlign = inBytes(getInt(Tok));
+ if (Error Err = split(Rest, ':', Split))
+ return Err;
+ unsigned ABIAlign;
+ if (Error Err = getIntInBytes(Tok, ABIAlign))
+ return Err;
if (AlignType != AGGREGATE_ALIGN && !ABIAlign)
- report_fatal_error(
+ return reportError(
"ABI alignment specification must be >0 for non-aggregate types");
if (!isUInt<16>(ABIAlign))
- report_fatal_error("Invalid ABI alignment, must be a 16bit integer");
+ return reportError("Invalid ABI alignment, must be a 16bit integer");
if (ABIAlign != 0 && !isPowerOf2_64(ABIAlign))
- report_fatal_error("Invalid ABI alignment, must be a power of 2");
+ return reportError("Invalid ABI alignment, must be a power of 2");
// Preferred alignment.
unsigned PrefAlign = ABIAlign;
if (!Rest.empty()) {
- Split = split(Rest, ':');
- PrefAlign = inBytes(getInt(Tok));
+ if (Error Err = split(Rest, ':', Split))
+ return Err;
+ if (Error Err = getIntInBytes(Tok, PrefAlign))
+ return Err;
}
if (!isUInt<16>(PrefAlign))
- report_fatal_error(
+ return reportError(
"Invalid preferred alignment, must be a 16bit integer");
if (PrefAlign != 0 && !isPowerOf2_64(PrefAlign))
- report_fatal_error("Invalid preferred alignment, must be a power of 2");
+ return reportError("Invalid preferred alignment, must be a power of 2");
- setAlignment(AlignType, assumeAligned(ABIAlign), assumeAligned(PrefAlign),
- Size);
+ if (Error Err = setAlignment(AlignType, assumeAligned(ABIAlign),
+ assumeAligned(PrefAlign), Size))
+ return Err;
break;
}
case 'n': // Native integer types.
while (true) {
- unsigned Width = getInt(Tok);
+ unsigned Width;
+ if (Error Err = getInt(Tok, Width))
+ return Err;
if (Width == 0)
- report_fatal_error(
+ return reportError(
"Zero width native integer type in datalayout string");
LegalIntWidths.push_back(Width);
if (Rest.empty())
break;
- Split = split(Rest, ':');
+ if (Error Err = split(Rest, ':', Split))
+ return Err;
}
break;
case 'S': { // Stack natural alignment.
- uint64_t Alignment = inBytes(getInt(Tok));
+ uint64_t Alignment;
+ if (Error Err = getIntInBytes(Tok, Alignment))
+ return Err;
if (Alignment != 0 && !llvm::isPowerOf2_64(Alignment))
- report_fatal_error("Alignment is neither 0 nor a power of 2");
+ return reportError("Alignment is neither 0 nor a power of 2");
StackNaturalAlign = MaybeAlign(Alignment);
break;
}
@@ -403,34 +459,44 @@ void DataLayout::parseSpecifier(StringRef Desc) {
TheFunctionPtrAlignType = FunctionPtrAlignType::MultipleOfFunctionAlign;
break;
default:
- report_fatal_error("Unknown function pointer alignment type in "
+ return reportError("Unknown function pointer alignment type in "
"datalayout string");
}
Tok = Tok.substr(1);
- uint64_t Alignment = inBytes(getInt(Tok));
+ uint64_t Alignment;
+ if (Error Err = getIntInBytes(Tok, Alignment))
+ return Err;
if (Alignment != 0 && !llvm::isPowerOf2_64(Alignment))
- report_fatal_error("Alignment is neither 0 nor a power of 2");
+ return reportError("Alignment is neither 0 nor a power of 2");
FunctionPtrAlign = MaybeAlign(Alignment);
break;
}
case 'P': { // Function address space.
- ProgramAddrSpace = getAddrSpace(Tok);
+ if (Error Err = getAddrSpace(Tok, ProgramAddrSpace))
+ return Err;
break;
}
case 'A': { // Default stack/alloca address space.
- AllocaAddrSpace = getAddrSpace(Tok);
+ if (Error Err = getAddrSpace(Tok, AllocaAddrSpace))
+ return Err;
+ break;
+ }
+ case 'G': { // Default address space for global variables.
+ if (Error Err = getAddrSpace(Tok, DefaultGlobalsAddrSpace))
+ return Err;
break;
}
case 'm':
if (!Tok.empty())
- report_fatal_error("Unexpected trailing characters after mangling specifier in datalayout string");
+ return reportError("Unexpected trailing characters after mangling "
+ "specifier in datalayout string");
if (Rest.empty())
- report_fatal_error("Expected mangling specifier in datalayout string");
+ return reportError("Expected mangling specifier in datalayout string");
if (Rest.size() > 1)
- report_fatal_error("Unknown mangling specifier in datalayout string");
+ return reportError("Unknown mangling specifier in datalayout string");
switch(Rest[0]) {
default:
- report_fatal_error("Unknown mangling in datalayout string");
+ return reportError("Unknown mangling in datalayout string");
case 'e':
ManglingMode = MM_ELF;
break;
@@ -452,10 +518,12 @@ void DataLayout::parseSpecifier(StringRef Desc) {
}
break;
default:
- report_fatal_error("Unknown specifier in datalayout string");
+ return reportError("Unknown specifier in datalayout string");
break;
}
}
+
+ return Error::success();
}
DataLayout::DataLayout(const Module *M) {
@@ -469,6 +537,7 @@ bool DataLayout::operator==(const DataLayout &Other) const {
AllocaAddrSpace == Other.AllocaAddrSpace &&
StackNaturalAlign == Other.StackNaturalAlign &&
ProgramAddrSpace == Other.ProgramAddrSpace &&
+ DefaultGlobalsAddrSpace == Other.DefaultGlobalsAddrSpace &&
FunctionPtrAlign == Other.FunctionPtrAlign &&
TheFunctionPtrAlignType == Other.TheFunctionPtrAlignType &&
ManglingMode == Other.ManglingMode &&
@@ -487,17 +556,17 @@ DataLayout::findAlignmentLowerBound(AlignTypeEnum AlignType,
});
}
-void DataLayout::setAlignment(AlignTypeEnum align_type, Align abi_align,
- Align pref_align, uint32_t bit_width) {
+Error DataLayout::setAlignment(AlignTypeEnum align_type, Align abi_align,
+ Align pref_align, uint32_t bit_width) {
// AlignmentsTy::ABIAlign and AlignmentsTy::PrefAlign were once stored as
// uint16_t, it is unclear if there are requirements for alignment to be less
// than 2^16 other than storage. In the meantime we leave the restriction as
// an assert. See D67400 for context.
assert(Log2(abi_align) < 16 && Log2(pref_align) < 16 && "Alignment too big");
if (!isUInt<24>(bit_width))
- report_fatal_error("Invalid bit width, must be a 24bit integer");
+ return reportError("Invalid bit width, must be a 24bit integer");
if (pref_align < abi_align)
- report_fatal_error(
+ return reportError(
"Preferred alignment cannot be less than the ABI alignment");
AlignmentsTy::iterator I = findAlignmentLowerBound(align_type, bit_width);
@@ -511,24 +580,35 @@ void DataLayout::setAlignment(AlignTypeEnum align_type, Align abi_align,
Alignments.insert(I, LayoutAlignElem::get(align_type, abi_align,
pref_align, bit_width));
}
-}
+ return Error::success();
+}
+
+const PointerAlignElem &
+DataLayout::getPointerAlignElem(uint32_t AddressSpace) const {
+ if (AddressSpace != 0) {
+ auto I = lower_bound(Pointers, AddressSpace,
+ [](const PointerAlignElem &A, uint32_t AddressSpace) {
+ return A.AddressSpace < AddressSpace;
+ });
+ if (I != Pointers.end() && I->AddressSpace == AddressSpace)
+ return *I;
+ }
-DataLayout::PointersTy::iterator
-DataLayout::findPointerLowerBound(uint32_t AddressSpace) {
- return std::lower_bound(Pointers.begin(), Pointers.end(), AddressSpace,
- [](const PointerAlignElem &A, uint32_t AddressSpace) {
- return A.AddressSpace < AddressSpace;
- });
+ assert(Pointers[0].AddressSpace == 0);
+ return Pointers[0];
}
-void DataLayout::setPointerAlignment(uint32_t AddrSpace, Align ABIAlign,
- Align PrefAlign, uint32_t TypeByteWidth,
- uint32_t IndexWidth) {
+Error DataLayout::setPointerAlignment(uint32_t AddrSpace, Align ABIAlign,
+ Align PrefAlign, uint32_t TypeByteWidth,
+ uint32_t IndexWidth) {
if (PrefAlign < ABIAlign)
- report_fatal_error(
+ return reportError(
"Preferred alignment cannot be less than the ABI alignment");
- PointersTy::iterator I = findPointerLowerBound(AddrSpace);
+ auto I = lower_bound(Pointers, AddrSpace,
+ [](const PointerAlignElem &A, uint32_t AddressSpace) {
+ return A.AddressSpace < AddressSpace;
+ });
if (I == Pointers.end() || I->AddressSpace != AddrSpace) {
Pointers.insert(I, PointerAlignElem::get(AddrSpace, ABIAlign, PrefAlign,
TypeByteWidth, IndexWidth));
@@ -538,49 +618,19 @@ void DataLayout::setPointerAlignment(uint32_t AddrSpace, Align ABIAlign,
I->TypeByteWidth = TypeByteWidth;
I->IndexWidth = IndexWidth;
}
+ return Error::success();
}
-/// getAlignmentInfo - Return the alignment (either ABI if ABIInfo = true or
-/// preferred if ABIInfo = false) the layout wants for the specified datatype.
-Align DataLayout::getAlignmentInfo(AlignTypeEnum AlignType, uint32_t BitWidth,
- bool ABIInfo, Type *Ty) const {
- AlignmentsTy::const_iterator I = findAlignmentLowerBound(AlignType, BitWidth);
- // See if we found an exact match. Of if we are looking for an integer type,
- // but don't have an exact match take the next largest integer. This is where
- // the lower_bound will point to when it fails an exact match.
- if (I != Alignments.end() && I->AlignType == (unsigned)AlignType &&
- (I->TypeBitWidth == BitWidth || AlignType == INTEGER_ALIGN))
- return ABIInfo ? I->ABIAlign : I->PrefAlign;
-
- if (AlignType == INTEGER_ALIGN) {
- // If we didn't have a larger value try the largest value we have.
- if (I != Alignments.begin()) {
- --I; // Go to the previous entry and see if its an integer.
- if (I->AlignType == INTEGER_ALIGN)
- return ABIInfo ? I->ABIAlign : I->PrefAlign;
- }
- } else if (AlignType == VECTOR_ALIGN) {
- // By default, use natural alignment for vector types. This is consistent
- // with what clang and llvm-gcc do.
- unsigned Alignment =
- getTypeAllocSize(cast<VectorType>(Ty)->getElementType());
- // We're only calculating a natural alignment, so it doesn't have to be
- // based on the full size for scalable vectors. Using the minimum element
- // count should be enough here.
- Alignment *= cast<VectorType>(Ty)->getElementCount().Min;
- Alignment = PowerOf2Ceil(Alignment);
- return Align(Alignment);
- }
-
- // If we still couldn't find a reasonable default alignment, fall back
- // to a simple heuristic that the alignment is the first power of two
- // greater-or-equal to the store size of the type. This is a reasonable
- // approximation of reality, and if the user wanted something less
- // less conservative, they should have specified it explicitly in the data
- // layout.
- unsigned Alignment = getTypeStoreSize(Ty);
- Alignment = PowerOf2Ceil(Alignment);
- return Align(Alignment);
+Align DataLayout::getIntegerAlignment(uint32_t BitWidth,
+ bool abi_or_pref) const {
+ auto I = findAlignmentLowerBound(INTEGER_ALIGN, BitWidth);
+ // If we don't have an exact match, use alignment of next larger integer
+ // type. If there is none, use alignment of largest integer type by going
+ // back one element.
+ if (I == Alignments.end() || I->AlignType != INTEGER_ALIGN)
+ --I;
+ assert(I->AlignType == INTEGER_ALIGN && "Must be integer alignment");
+ return abi_or_pref ? I->ABIAlign : I->PrefAlign;
}
namespace {
@@ -642,30 +692,15 @@ const StructLayout *DataLayout::getStructLayout(StructType *Ty) const {
}
Align DataLayout::getPointerABIAlignment(unsigned AS) const {
- PointersTy::const_iterator I = findPointerLowerBound(AS);
- if (I == Pointers.end() || I->AddressSpace != AS) {
- I = findPointerLowerBound(0);
- assert(I->AddressSpace == 0);
- }
- return I->ABIAlign;
+ return getPointerAlignElem(AS).ABIAlign;
}
Align DataLayout::getPointerPrefAlignment(unsigned AS) const {
- PointersTy::const_iterator I = findPointerLowerBound(AS);
- if (I == Pointers.end() || I->AddressSpace != AS) {
- I = findPointerLowerBound(0);
- assert(I->AddressSpace == 0);
- }
- return I->PrefAlign;
+ return getPointerAlignElem(AS).PrefAlign;
}
unsigned DataLayout::getPointerSize(unsigned AS) const {
- PointersTy::const_iterator I = findPointerLowerBound(AS);
- if (I == Pointers.end() || I->AddressSpace != AS) {
- I = findPointerLowerBound(0);
- assert(I->AddressSpace == 0);
- }
- return I->TypeByteWidth;
+ return getPointerAlignElem(AS).TypeByteWidth;
}
unsigned DataLayout::getMaxPointerSize() const {
@@ -684,12 +719,7 @@ unsigned DataLayout::getPointerTypeSizeInBits(Type *Ty) const {
}
unsigned DataLayout::getIndexSize(unsigned AS) const {
- PointersTy::const_iterator I = findPointerLowerBound(AS);
- if (I == Pointers.end() || I->AddressSpace != AS) {
- I = findPointerLowerBound(0);
- assert(I->AddressSpace == 0);
- }
- return I->IndexWidth;
+ return getPointerAlignElem(AS).IndexWidth;
}
unsigned DataLayout::getIndexTypeSizeInBits(Type *Ty) const {
@@ -708,8 +738,6 @@ unsigned DataLayout::getIndexTypeSizeInBits(Type *Ty) const {
== false) for the requested type \a Ty.
*/
Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
- AlignTypeEnum AlignType;
-
assert(Ty->isSized() && "Cannot getTypeInfo() on a type that is unsized!");
switch (Ty->getTypeID()) {
// Early escape for the non-numeric types.
@@ -730,12 +758,15 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
// Get the layout annotation... which is lazily created on demand.
const StructLayout *Layout = getStructLayout(cast<StructType>(Ty));
- const Align Align = getAlignmentInfo(AGGREGATE_ALIGN, 0, abi_or_pref, Ty);
+ const LayoutAlignElem &AggregateAlign = Alignments[0];
+ assert(AggregateAlign.AlignType == AGGREGATE_ALIGN &&
+ "Aggregate alignment must be first alignment entry");
+ const Align Align =
+ abi_or_pref ? AggregateAlign.ABIAlign : AggregateAlign.PrefAlign;
return std::max(Align, Layout->getAlignment());
}
case Type::IntegerTyID:
- AlignType = INTEGER_ALIGN;
- break;
+ return getIntegerAlignment(Ty->getIntegerBitWidth(), abi_or_pref);
case Type::HalfTyID:
case Type::BFloatTyID:
case Type::FloatTyID:
@@ -744,22 +775,47 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
// same size and alignment, so they look the same here.
case Type::PPC_FP128TyID:
case Type::FP128TyID:
- case Type::X86_FP80TyID:
- AlignType = FLOAT_ALIGN;
- break;
+ case Type::X86_FP80TyID: {
+ unsigned BitWidth = getTypeSizeInBits(Ty).getFixedSize();
+ auto I = findAlignmentLowerBound(FLOAT_ALIGN, BitWidth);
+ if (I != Alignments.end() && I->AlignType == FLOAT_ALIGN &&
+ I->TypeBitWidth == BitWidth)
+ return abi_or_pref ? I->ABIAlign : I->PrefAlign;
+
+ // If we still couldn't find a reasonable default alignment, fall back
+ // to a simple heuristic that the alignment is the first power of two
+ // greater-or-equal to the store size of the type. This is a reasonable
+ // approximation of reality, and if the user wanted something less
+ // less conservative, they should have specified it explicitly in the data
+ // layout.
+ return Align(PowerOf2Ceil(BitWidth / 8));
+ }
case Type::X86_MMXTyID:
case Type::FixedVectorTyID:
- case Type::ScalableVectorTyID:
- AlignType = VECTOR_ALIGN;
- break;
+ case Type::ScalableVectorTyID: {
+ unsigned BitWidth = getTypeSizeInBits(Ty).getKnownMinSize();
+ auto I = findAlignmentLowerBound(VECTOR_ALIGN, BitWidth);
+ if (I != Alignments.end() && I->AlignType == VECTOR_ALIGN &&
+ I->TypeBitWidth == BitWidth)
+ return abi_or_pref ? I->ABIAlign : I->PrefAlign;
+
+ // By default, use natural alignment for vector types. This is consistent
+ // with what clang and llvm-gcc do.
+ // TODO: This should probably not be using the alloc size.
+ unsigned Alignment =
+ getTypeAllocSize(cast<VectorType>(Ty)->getElementType());
+ // We're only calculating a natural alignment, so it doesn't have to be
+ // based on the full size for scalable vectors. Using the minimum element
+ // count should be enough here.
+ Alignment *= cast<VectorType>(Ty)->getElementCount().getKnownMinValue();
+ Alignment = PowerOf2Ceil(Alignment);
+ return Align(Alignment);
+ }
+ case Type::X86_AMXTyID:
+ return Align(64);
default:
llvm_unreachable("Bad type for getAlignment!!!");
}
-
- // If we're dealing with a scalable vector, we just need the known minimum
- // size for determining alignment. If not, we'll get the exact size.
- return getAlignmentInfo(AlignType, getTypeSizeInBits(Ty).getKnownMinSize(),
- abi_or_pref, Ty);
}
/// TODO: Remove this function once the transition to Align is over.
@@ -771,12 +827,6 @@ Align DataLayout::getABITypeAlign(Type *Ty) const {
return getAlignment(Ty, true);
}
-/// getABIIntegerTypeAlignment - Return the minimum ABI-required alignment for
-/// an integer type of the specified bitwidth.
-Align DataLayout::getABIIntegerTypeAlignment(unsigned BitWidth) const {
- return getAlignmentInfo(INTEGER_ALIGN, BitWidth, true, nullptr);
-}
-
/// TODO: Remove this function once the transition to Align is over.
unsigned DataLayout::getPrefTypeAlignment(Type *Ty) const {
return getPrefTypeAlign(Ty).value();
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index 190b220dc9aa..d7656b9dd1f8 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -652,7 +652,8 @@ bool llvm::stripNonLineTableDebugInfo(Module &M) {
MDNode *InlinedAt = DL.getInlinedAt();
Scope = remap(Scope);
InlinedAt = remap(InlinedAt);
- return DebugLoc::get(DL.getLine(), DL.getCol(), Scope, InlinedAt);
+ return DILocation::get(M.getContext(), DL.getLine(), DL.getCol(),
+ Scope, InlinedAt);
};
if (I.getDebugLoc() != DebugLoc())
@@ -696,6 +697,38 @@ void Instruction::applyMergedLocation(const DILocation *LocA,
setDebugLoc(DILocation::getMergedLocation(LocA, LocB));
}
+void Instruction::updateLocationAfterHoist() { dropLocation(); }
+
+void Instruction::dropLocation() {
+ const DebugLoc &DL = getDebugLoc();
+ if (!DL)
+ return;
+
+ // If this isn't a call, drop the location to allow a location from a
+ // preceding instruction to propagate.
+ if (!isa<CallBase>(this)) {
+ setDebugLoc(DebugLoc());
+ return;
+ }
+
+ // Set a line 0 location for calls to preserve scope information in case
+ // inlining occurs.
+ DISubprogram *SP = getFunction()->getSubprogram();
+ if (SP)
+ // If a function scope is available, set it on the line 0 location. When
+ // hoisting a call to a predecessor block, using the function scope avoids
+ // making it look like the callee was reached earlier than it should be.
+ setDebugLoc(DILocation::get(getContext(), 0, 0, SP));
+ else
+ // The parent function has no scope. Go ahead and drop the location. If
+ // the parent function is inlined, and the callee has a subprogram, the
+ // inliner will attach a location to the call.
+ //
+ // One alternative is to set a line 0 location with the existing scope and
+ // inlinedAt info. The location might be sensitive to when inlining occurs.
+ setDebugLoc(DebugLoc());
+}
+
//===----------------------------------------------------------------------===//
// LLVM C API implementations.
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp
index 110d94116f10..77bba9f7ed0e 100644
--- a/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -133,7 +133,7 @@ const DILocation *DILocation::getMergedLocation(const DILocation *LocA,
}
Optional<unsigned> DILocation::encodeDiscriminator(unsigned BD, unsigned DF, unsigned CI) {
- SmallVector<unsigned, 3> Components = {BD, DF, CI};
+ std::array<unsigned, 3> Components = {BD, DF, CI};
uint64_t RemainingWork = 0U;
// We use RemainingWork to figure out if we have no remaining components to
// encode. For example: if BD != 0 but DF == 0 && CI == 0, we don't need to
@@ -435,6 +435,84 @@ DISubrange::BoundType DISubrange::getStride() const {
return BoundType();
}
+DIGenericSubrange *DIGenericSubrange::getImpl(LLVMContext &Context,
+ Metadata *CountNode, Metadata *LB,
+ Metadata *UB, Metadata *Stride,
+ StorageType Storage,
+ bool ShouldCreate) {
+ DEFINE_GETIMPL_LOOKUP(DIGenericSubrange, (CountNode, LB, UB, Stride));
+ Metadata *Ops[] = {CountNode, LB, UB, Stride};
+ DEFINE_GETIMPL_STORE_NO_CONSTRUCTOR_ARGS(DIGenericSubrange, Ops);
+}
+
+DIGenericSubrange::BoundType DIGenericSubrange::getCount() const {
+ Metadata *CB = getRawCountNode();
+ if (!CB)
+ return BoundType();
+
+ assert((isa<DIVariable>(CB) || isa<DIExpression>(CB)) &&
+ "Count must be signed constant or DIVariable or DIExpression");
+
+ if (auto *MD = dyn_cast<DIVariable>(CB))
+ return BoundType(MD);
+
+ if (auto *MD = dyn_cast<DIExpression>(CB))
+ return BoundType(MD);
+
+ return BoundType();
+}
+
+DIGenericSubrange::BoundType DIGenericSubrange::getLowerBound() const {
+ Metadata *LB = getRawLowerBound();
+ if (!LB)
+ return BoundType();
+
+ assert((isa<DIVariable>(LB) || isa<DIExpression>(LB)) &&
+ "LowerBound must be signed constant or DIVariable or DIExpression");
+
+ if (auto *MD = dyn_cast<DIVariable>(LB))
+ return BoundType(MD);
+
+ if (auto *MD = dyn_cast<DIExpression>(LB))
+ return BoundType(MD);
+
+ return BoundType();
+}
+
+DIGenericSubrange::BoundType DIGenericSubrange::getUpperBound() const {
+ Metadata *UB = getRawUpperBound();
+ if (!UB)
+ return BoundType();
+
+ assert((isa<DIVariable>(UB) || isa<DIExpression>(UB)) &&
+ "UpperBound must be signed constant or DIVariable or DIExpression");
+
+ if (auto *MD = dyn_cast<DIVariable>(UB))
+ return BoundType(MD);
+
+ if (auto *MD = dyn_cast<DIExpression>(UB))
+ return BoundType(MD);
+
+ return BoundType();
+}
+
+DIGenericSubrange::BoundType DIGenericSubrange::getStride() const {
+ Metadata *ST = getRawStride();
+ if (!ST)
+ return BoundType();
+
+ assert((isa<DIVariable>(ST) || isa<DIExpression>(ST)) &&
+ "Stride must be signed constant or DIVariable or DIExpression");
+
+ if (auto *MD = dyn_cast<DIVariable>(ST))
+ return BoundType(MD);
+
+ if (auto *MD = dyn_cast<DIExpression>(ST))
+ return BoundType(MD);
+
+ return BoundType();
+}
+
DIEnumerator *DIEnumerator::getImpl(LLVMContext &Context, const APInt &Value,
bool IsUnsigned, MDString *Name,
StorageType Storage, bool ShouldCreate) {
@@ -470,6 +548,20 @@ Optional<DIBasicType::Signedness> DIBasicType::getSignedness() const {
}
}
+DIStringType *DIStringType::getImpl(LLVMContext &Context, unsigned Tag,
+ MDString *Name, Metadata *StringLength,
+ Metadata *StringLengthExp,
+ uint64_t SizeInBits, uint32_t AlignInBits,
+ unsigned Encoding, StorageType Storage,
+ bool ShouldCreate) {
+ assert(isCanonical(Name) && "Expected canonical MDString");
+ DEFINE_GETIMPL_LOOKUP(DIStringType, (Tag, Name, StringLength, StringLengthExp,
+ SizeInBits, AlignInBits, Encoding));
+ Metadata *Ops[] = {nullptr, nullptr, Name, StringLength, StringLengthExp};
+ DEFINE_GETIMPL_STORE(DIStringType, (Tag, SizeInBits, AlignInBits, Encoding),
+ Ops);
+}
+
DIDerivedType *DIDerivedType::getImpl(
LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *File,
unsigned Line, Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits,
@@ -493,18 +585,20 @@ DICompositeType *DICompositeType::getImpl(
uint32_t AlignInBits, uint64_t OffsetInBits, DIFlags Flags,
Metadata *Elements, unsigned RuntimeLang, Metadata *VTableHolder,
Metadata *TemplateParams, MDString *Identifier, Metadata *Discriminator,
- Metadata *DataLocation, StorageType Storage, bool ShouldCreate) {
+ Metadata *DataLocation, Metadata *Associated, Metadata *Allocated,
+ Metadata *Rank, StorageType Storage, bool ShouldCreate) {
assert(isCanonical(Name) && "Expected canonical MDString");
// Keep this in sync with buildODRType.
- DEFINE_GETIMPL_LOOKUP(DICompositeType,
- (Tag, Name, File, Line, Scope, BaseType, SizeInBits,
- AlignInBits, OffsetInBits, Flags, Elements,
- RuntimeLang, VTableHolder, TemplateParams, Identifier,
- Discriminator, DataLocation));
+ DEFINE_GETIMPL_LOOKUP(
+ DICompositeType,
+ (Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
+ OffsetInBits, Flags, Elements, RuntimeLang, VTableHolder, TemplateParams,
+ Identifier, Discriminator, DataLocation, Associated, Allocated, Rank));
Metadata *Ops[] = {File, Scope, Name, BaseType,
Elements, VTableHolder, TemplateParams, Identifier,
- Discriminator, DataLocation};
+ Discriminator, DataLocation, Associated, Allocated,
+ Rank};
DEFINE_GETIMPL_STORE(DICompositeType, (Tag, Line, RuntimeLang, SizeInBits,
AlignInBits, OffsetInBits, Flags),
Ops);
@@ -516,7 +610,8 @@ DICompositeType *DICompositeType::buildODRType(
uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
DIFlags Flags, Metadata *Elements, unsigned RuntimeLang,
Metadata *VTableHolder, Metadata *TemplateParams, Metadata *Discriminator,
- Metadata *DataLocation) {
+ Metadata *DataLocation, Metadata *Associated, Metadata *Allocated,
+ Metadata *Rank) {
assert(!Identifier.getString().empty() && "Expected valid identifier");
if (!Context.isODRUniquingDebugTypes())
return nullptr;
@@ -526,7 +621,7 @@ DICompositeType *DICompositeType::buildODRType(
Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang,
VTableHolder, TemplateParams, &Identifier, Discriminator,
- DataLocation);
+ DataLocation, Associated, Allocated, Rank);
// Only mutate CT if it's a forward declaration and the new operands aren't.
assert(CT->getRawIdentifier() == &Identifier && "Wrong ODR identifier?");
@@ -538,7 +633,8 @@ DICompositeType *DICompositeType::buildODRType(
Flags);
Metadata *Ops[] = {File, Scope, Name, BaseType,
Elements, VTableHolder, TemplateParams, &Identifier,
- Discriminator, DataLocation};
+ Discriminator, DataLocation, Associated, Allocated,
+ Rank};
assert((std::end(Ops) - std::begin(Ops)) == (int)CT->getNumOperands() &&
"Mismatched number of operands");
for (unsigned I = 0, E = CT->getNumOperands(); I != E; ++I)
@@ -553,7 +649,8 @@ DICompositeType *DICompositeType::getODRType(
uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
DIFlags Flags, Metadata *Elements, unsigned RuntimeLang,
Metadata *VTableHolder, Metadata *TemplateParams, Metadata *Discriminator,
- Metadata *DataLocation) {
+ Metadata *DataLocation, Metadata *Associated, Metadata *Allocated,
+ Metadata *Rank) {
assert(!Identifier.getString().empty() && "Expected valid identifier");
if (!Context.isODRUniquingDebugTypes())
return nullptr;
@@ -562,7 +659,8 @@ DICompositeType *DICompositeType::getODRType(
CT = DICompositeType::getDistinct(
Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, VTableHolder,
- TemplateParams, &Identifier, Discriminator, DataLocation);
+ TemplateParams, &Identifier, Discriminator, DataLocation, Associated,
+ Allocated, Rank);
return CT;
}
@@ -828,14 +926,14 @@ DIModule *DIModule::getImpl(LLVMContext &Context, Metadata *File,
Metadata *Scope, MDString *Name,
MDString *ConfigurationMacros,
MDString *IncludePath, MDString *APINotesFile,
- unsigned LineNo, StorageType Storage,
+ unsigned LineNo, bool IsDecl, StorageType Storage,
bool ShouldCreate) {
assert(isCanonical(Name) && "Expected canonical MDString");
DEFINE_GETIMPL_LOOKUP(DIModule, (File, Scope, Name, ConfigurationMacros,
- IncludePath, APINotesFile, LineNo));
+ IncludePath, APINotesFile, LineNo, IsDecl));
Metadata *Ops[] = {File, Scope, Name, ConfigurationMacros,
IncludePath, APINotesFile};
- DEFINE_GETIMPL_STORE(DIModule, (LineNo), Ops);
+ DEFINE_GETIMPL_STORE(DIModule, (LineNo, IsDecl), Ops);
}
DITemplateTypeParameter *
@@ -1016,6 +1114,7 @@ bool DIExpression::isValid() const {
return I->get() == expr_op_begin()->get() && I->getArg(0) == 1 &&
getNumElements() == 2;
}
+ case dwarf::DW_OP_LLVM_implicit_pointer:
case dwarf::DW_OP_LLVM_convert:
case dwarf::DW_OP_LLVM_tag_offset:
case dwarf::DW_OP_constu:
@@ -1040,6 +1139,8 @@ bool DIExpression::isValid() const {
case dwarf::DW_OP_regx:
case dwarf::DW_OP_bregx:
case dwarf::DW_OP_push_object_address:
+ case dwarf::DW_OP_over:
+ case dwarf::DW_OP_consts:
break;
}
}
@@ -1313,6 +1414,15 @@ bool DIExpression::isConstant() const {
return true;
}
+bool DIExpression::isSignedConstant() const {
+ // Recognize DW_OP_consts C
+ if (getNumElements() != 2)
+ return false;
+ if (getElement(0) != dwarf::DW_OP_consts)
+ return false;
+ return true;
+}
+
DIExpression::ExtOps DIExpression::getExtOps(unsigned FromSize, unsigned ToSize,
bool Signed) {
dwarf::TypeKind TK = Signed ? dwarf::DW_ATE_signed : dwarf::DW_ATE_unsigned;
diff --git a/llvm/lib/IR/DebugLoc.cpp b/llvm/lib/IR/DebugLoc.cpp
index e945cbcba782..993f3a39e6ff 100644
--- a/llvm/lib/IR/DebugLoc.cpp
+++ b/llvm/lib/IR/DebugLoc.cpp
@@ -50,7 +50,7 @@ DebugLoc DebugLoc::getFnDebugLoc() const {
// FIXME: Add a method on \a DILocation that does this work.
const MDNode *Scope = getInlinedAtScope();
if (auto *SP = getDISubprogram(Scope))
- return DebugLoc::get(SP->getScopeLine(), 0, SP);
+ return DILocation::get(SP->getContext(), SP->getScopeLine(), 0, SP);
return DebugLoc();
}
@@ -68,21 +68,9 @@ void DebugLoc::setImplicitCode(bool ImplicitCode) {
}
}
-DebugLoc DebugLoc::get(unsigned Line, unsigned Col, const MDNode *Scope,
- const MDNode *InlinedAt, bool ImplicitCode) {
- // If no scope is available, this is an unknown location.
- if (!Scope)
- return DebugLoc();
-
- return DILocation::get(Scope->getContext(), Line, Col,
- const_cast<MDNode *>(Scope),
- const_cast<MDNode *>(InlinedAt), ImplicitCode);
-}
-
DebugLoc DebugLoc::appendInlinedAt(const DebugLoc &DL, DILocation *InlinedAt,
LLVMContext &Ctx,
- DenseMap<const MDNode *, MDNode *> &Cache,
- bool ReplaceLast) {
+ DenseMap<const MDNode *, MDNode *> &Cache) {
SmallVector<DILocation *, 3> InlinedAtLocations;
DILocation *Last = InlinedAt;
DILocation *CurInlinedAt = DL;
@@ -95,8 +83,6 @@ DebugLoc DebugLoc::appendInlinedAt(const DebugLoc &DL, DILocation *InlinedAt,
break;
}
- if (ReplaceLast && !IA->getInlinedAt())
- break;
InlinedAtLocations.push_back(IA);
CurInlinedAt = IA;
}
diff --git a/llvm/lib/IR/DiagnosticInfo.cpp b/llvm/lib/IR/DiagnosticInfo.cpp
index 6528c723fbfa..8820a79421c3 100644
--- a/llvm/lib/IR/DiagnosticInfo.cpp
+++ b/llvm/lib/IR/DiagnosticInfo.cpp
@@ -32,6 +32,7 @@
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Regex.h"
#include "llvm/Support/ScopedPrinter.h"
@@ -213,6 +214,20 @@ DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key,
unsigned long long N)
: Key(std::string(Key)), Val(utostr(N)) {}
+DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key,
+ ElementCount EC)
+ : Key(std::string(Key)) {
+ raw_string_ostream OS(Val);
+ EC.print(OS);
+}
+
+DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key,
+ InstructionCost C)
+ : Key(std::string(Key)) {
+ raw_string_ostream OS(Val);
+ C.print(OS);
+}
+
DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, DebugLoc Loc)
: Key(std::string(Key)), Loc(Loc) {
if (Loc) {
@@ -368,16 +383,5 @@ std::string DiagnosticInfoOptimizationBase::getMsg() const {
return OS.str();
}
-DiagnosticInfoMisExpect::DiagnosticInfoMisExpect(const Instruction *Inst,
- Twine &Msg)
- : DiagnosticInfoWithLocationBase(DK_MisExpect, DS_Warning,
- *Inst->getParent()->getParent(),
- Inst->getDebugLoc()),
- Msg(Msg) {}
-
-void DiagnosticInfoMisExpect::print(DiagnosticPrinter &DP) const {
- DP << getLocationStr() << ": " << getMsg();
-}
-
void OptimizationRemarkAnalysisFPCommute::anchor() {}
void OptimizationRemarkAnalysisAliasing::anchor() {}
diff --git a/llvm/lib/IR/Dominators.cpp b/llvm/lib/IR/Dominators.cpp
index bb1cc347dcb1..fbc28c202aec 100644
--- a/llvm/lib/IR/Dominators.cpp
+++ b/llvm/lib/IR/Dominators.cpp
@@ -90,9 +90,11 @@ template void llvm::DomTreeBuilder::DeleteEdge<DomTreeBuilder::BBPostDomTree>(
DomTreeBuilder::BBPostDomTree &DT, BasicBlock *From, BasicBlock *To);
template void llvm::DomTreeBuilder::ApplyUpdates<DomTreeBuilder::BBDomTree>(
- DomTreeBuilder::BBDomTree &DT, DomTreeBuilder::BBUpdates);
+ DomTreeBuilder::BBDomTree &DT, DomTreeBuilder::BBDomTreeGraphDiff &,
+ DomTreeBuilder::BBDomTreeGraphDiff *);
template void llvm::DomTreeBuilder::ApplyUpdates<DomTreeBuilder::BBPostDomTree>(
- DomTreeBuilder::BBPostDomTree &DT, DomTreeBuilder::BBUpdates);
+ DomTreeBuilder::BBPostDomTree &DT, DomTreeBuilder::BBPostDomTreeGraphDiff &,
+ DomTreeBuilder::BBPostDomTreeGraphDiff *);
template bool llvm::DomTreeBuilder::Verify<DomTreeBuilder::BBDomTree>(
const DomTreeBuilder::BBDomTree &DT,
@@ -113,8 +115,15 @@ bool DominatorTree::invalidate(Function &F, const PreservedAnalyses &PA,
// dominates - Return true if Def dominates a use in User. This performs
// the special checks necessary if Def and User are in the same basic block.
// Note that Def doesn't dominate a use in Def itself!
-bool DominatorTree::dominates(const Instruction *Def,
+bool DominatorTree::dominates(const Value *DefV,
const Instruction *User) const {
+ const Instruction *Def = dyn_cast<Instruction>(DefV);
+ if (!Def) {
+ assert((isa<Argument>(DefV) || isa<Constant>(DefV)) &&
+ "Should be called with an instruction, argument or constant");
+ return true; // Arguments and constants dominate everything.
+ }
+
const BasicBlock *UseBB = User->getParent();
const BasicBlock *DefBB = Def->getParent();
@@ -248,7 +257,14 @@ bool DominatorTree::dominates(const BasicBlockEdge &BBE, const Use &U) const {
return dominates(BBE, UseBB);
}
-bool DominatorTree::dominates(const Instruction *Def, const Use &U) const {
+bool DominatorTree::dominates(const Value *DefV, const Use &U) const {
+ const Instruction *Def = dyn_cast<Instruction>(DefV);
+ if (!Def) {
+ assert((isa<Argument>(DefV) || isa<Constant>(DefV)) &&
+ "Should be called with an instruction, argument or constant");
+ return true; // Arguments and constants dominate everything.
+ }
+
Instruction *UserInst = cast<Instruction>(U.getUser());
const BasicBlock *DefBB = Def->getParent();
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index 10d535e3ab11..17247123f87f 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -43,6 +43,7 @@
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/IR/IntrinsicsRISCV.h"
#include "llvm/IR/IntrinsicsS390.h"
+#include "llvm/IR/IntrinsicsVE.h"
#include "llvm/IR/IntrinsicsWebAssembly.h"
#include "llvm/IR/IntrinsicsX86.h"
#include "llvm/IR/IntrinsicsXCore.h"
@@ -86,9 +87,11 @@ void Argument::setParent(Function *parent) {
Parent = parent;
}
-bool Argument::hasNonNullAttr() const {
+bool Argument::hasNonNullAttr(bool AllowUndefOrPoison) const {
if (!getType()->isPointerTy()) return false;
- if (getParent()->hasParamAttribute(getArgNo(), Attribute::NonNull))
+ if (getParent()->hasParamAttribute(getArgNo(), Attribute::NonNull) &&
+ (AllowUndefOrPoison ||
+ getParent()->hasParamAttribute(getArgNo(), Attribute::NoUndef)))
return true;
else if (getDereferenceableBytes() > 0 &&
!NullPointerIsDefined(getParent(),
@@ -102,6 +105,12 @@ bool Argument::hasByValAttr() const {
return hasAttribute(Attribute::ByVal);
}
+bool Argument::hasByRefAttr() const {
+ if (!getType()->isPointerTy())
+ return false;
+ return hasAttribute(Attribute::ByRef);
+}
+
bool Argument::hasSwiftSelfAttr() const {
return getParent()->hasParamAttribute(getArgNo(), Attribute::SwiftSelf);
}
@@ -121,7 +130,7 @@ bool Argument::hasPreallocatedAttr() const {
return hasAttribute(Attribute::Preallocated);
}
-bool Argument::hasPassPointeeByValueAttr() const {
+bool Argument::hasPassPointeeByValueCopyAttr() const {
if (!getType()->isPointerTy()) return false;
AttributeList Attrs = getParent()->getAttributes();
return Attrs.hasParamAttribute(getArgNo(), Attribute::ByVal) ||
@@ -129,27 +138,54 @@ bool Argument::hasPassPointeeByValueAttr() const {
Attrs.hasParamAttribute(getArgNo(), Attribute::Preallocated);
}
-uint64_t Argument::getPassPointeeByValueCopySize(const DataLayout &DL) const {
- AttributeSet ParamAttrs
- = getParent()->getAttributes().getParamAttributes(getArgNo());
+bool Argument::hasPointeeInMemoryValueAttr() const {
+ if (!getType()->isPointerTy())
+ return false;
+ AttributeList Attrs = getParent()->getAttributes();
+ return Attrs.hasParamAttribute(getArgNo(), Attribute::ByVal) ||
+ Attrs.hasParamAttribute(getArgNo(), Attribute::StructRet) ||
+ Attrs.hasParamAttribute(getArgNo(), Attribute::InAlloca) ||
+ Attrs.hasParamAttribute(getArgNo(), Attribute::Preallocated) ||
+ Attrs.hasParamAttribute(getArgNo(), Attribute::ByRef);
+}
+/// For a byval, sret, inalloca, or preallocated parameter, get the in-memory
+/// parameter type.
+static Type *getMemoryParamAllocType(AttributeSet ParamAttrs, Type *ArgTy) {
// FIXME: All the type carrying attributes are mutually exclusive, so there
// should be a single query to get the stored type that handles any of them.
if (Type *ByValTy = ParamAttrs.getByValType())
- return DL.getTypeAllocSize(ByValTy);
+ return ByValTy;
+ if (Type *ByRefTy = ParamAttrs.getByRefType())
+ return ByRefTy;
if (Type *PreAllocTy = ParamAttrs.getPreallocatedType())
- return DL.getTypeAllocSize(PreAllocTy);
+ return PreAllocTy;
- // FIXME: inalloca always depends on pointee element type. It's also possible
- // for byval to miss it.
+ // FIXME: sret and inalloca always depends on pointee element type. It's also
+ // possible for byval to miss it.
if (ParamAttrs.hasAttribute(Attribute::InAlloca) ||
ParamAttrs.hasAttribute(Attribute::ByVal) ||
+ ParamAttrs.hasAttribute(Attribute::StructRet) ||
ParamAttrs.hasAttribute(Attribute::Preallocated))
- return DL.getTypeAllocSize(cast<PointerType>(getType())->getElementType());
+ return cast<PointerType>(ArgTy)->getElementType();
+ return nullptr;
+}
+
+uint64_t Argument::getPassPointeeByValueCopySize(const DataLayout &DL) const {
+ AttributeSet ParamAttrs =
+ getParent()->getAttributes().getParamAttributes(getArgNo());
+ if (Type *MemTy = getMemoryParamAllocType(ParamAttrs, getType()))
+ return DL.getTypeAllocSize(MemTy);
return 0;
}
+Type *Argument::getPointeeInMemoryValueType() const {
+ AttributeSet ParamAttrs =
+ getParent()->getAttributes().getParamAttributes(getArgNo());
+ return getMemoryParamAllocType(ParamAttrs, getType());
+}
+
unsigned Argument::getParamAlignment() const {
assert(getType()->isPointerTy() && "Only pointers have alignments");
return getParent()->getParamAlignment(getArgNo());
@@ -165,6 +201,16 @@ Type *Argument::getParamByValType() const {
return getParent()->getParamByValType(getArgNo());
}
+Type *Argument::getParamStructRetType() const {
+ assert(getType()->isPointerTy() && "Only pointers have sret types");
+ return getParent()->getParamStructRetType(getArgNo());
+}
+
+Type *Argument::getParamByRefType() const {
+ assert(getType()->isPointerTy() && "Only pointers have byval types");
+ return getParent()->getParamByRefType(getArgNo());
+}
+
uint64_t Argument::getDereferenceableBytes() const {
assert(getType()->isPointerTy() &&
"Only pointers have dereferenceable bytes");
@@ -534,6 +580,21 @@ void Function::addDereferenceableOrNullParamAttr(unsigned ArgNo,
setAttributes(PAL);
}
+DenormalMode Function::getDenormalMode(const fltSemantics &FPType) const {
+ if (&FPType == &APFloat::IEEEsingle()) {
+ Attribute Attr = getFnAttribute("denormal-fp-math-f32");
+ StringRef Val = Attr.getValueAsString();
+ if (!Val.empty())
+ return parseDenormalFPAttribute(Val);
+
+ // If the f32 variant of the attribute isn't specified, try to use the
+ // generic one.
+ }
+
+ Attribute Attr = getFnAttribute("denormal-fp-math");
+ return parseDenormalFPAttribute(Attr.getValueAsString());
+}
+
const std::string &Function::getGC() const {
assert(hasGC() && "Function has no collector");
return getContext().getGC(*this);
@@ -551,6 +612,12 @@ void Function::clearGC() {
setValueSubclassDataBit(14, false);
}
+bool Function::hasStackProtectorFnAttr() const {
+ return hasFnAttribute(Attribute::StackProtect) ||
+ hasFnAttribute(Attribute::StackProtectStrong) ||
+ hasFnAttribute(Attribute::StackProtectReq);
+}
+
/// Copy all additional attributes (those not needed to create a Function) from
/// the Function Src to this one.
void Function::copyAttributesFrom(const Function *Src) {
@@ -582,6 +649,14 @@ static const char * const IntrinsicNameTable[] = {
#include "llvm/IR/IntrinsicImpl.inc"
#undef GET_INTRINSIC_TARGET_DATA
+bool Function::isTargetIntrinsic(Intrinsic::ID IID) {
+ return IID > TargetInfos[0].Count;
+}
+
+bool Function::isTargetIntrinsic() const {
+ return isTargetIntrinsic(IntID);
+}
+
/// Find the segment of \c IntrinsicNameTable for intrinsics with the same
/// target as \c Name, or the generic table if \c Name is not target specific.
///
@@ -674,9 +749,10 @@ static std::string getMangledTypeStr(Type* Ty) {
Result += "f";
} else if (VectorType* VTy = dyn_cast<VectorType>(Ty)) {
ElementCount EC = VTy->getElementCount();
- if (EC.Scalable)
+ if (EC.isScalable())
Result += "nx";
- Result += "v" + utostr(EC.Min) + getMangledTypeStr(VTy->getElementType());
+ Result += "v" + utostr(EC.getKnownMinValue()) +
+ getMangledTypeStr(VTy->getElementType());
} else if (Ty) {
switch (Ty->getTypeID()) {
default: llvm_unreachable("Unhandled type");
@@ -690,6 +766,7 @@ static std::string getMangledTypeStr(Type* Ty) {
case Type::FP128TyID: Result += "f128"; break;
case Type::PPC_FP128TyID: Result += "ppcf128"; break;
case Type::X86_MMXTyID: Result += "x86mmx"; break;
+ case Type::X86_AMXTyID: Result += "x86amx"; break;
case Type::IntegerTyID:
Result += "i" + utostr(cast<IntegerType>(Ty)->getBitWidth());
break;
@@ -707,6 +784,8 @@ StringRef Intrinsic::getName(ID id) {
std::string Intrinsic::getName(ID id, ArrayRef<Type*> Tys) {
assert(id < num_intrinsics && "Invalid intrinsic ID!");
+ assert((Tys.empty() || Intrinsic::isOverloaded(id)) &&
+ "This version of getName is for overloaded intrinsics only");
std::string Result(IntrinsicNameTable[id]);
for (Type *Ty : Tys) {
Result += "." + getMangledTypeStr(Ty);
@@ -770,7 +849,10 @@ enum IIT_Info {
IIT_SUBDIVIDE4_ARG = 45,
IIT_VEC_OF_BITCASTS_TO_INT = 46,
IIT_V128 = 47,
- IIT_BF16 = 48
+ IIT_BF16 = 48,
+ IIT_STRUCT9 = 49,
+ IIT_V256 = 50,
+ IIT_AMX = 51
};
static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
@@ -793,6 +875,9 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
case IIT_MMX:
OutputTable.push_back(IITDescriptor::get(IITDescriptor::MMX, 0));
return;
+ case IIT_AMX:
+ OutputTable.push_back(IITDescriptor::get(IITDescriptor::AMX, 0));
+ return;
case IIT_TOKEN:
OutputTable.push_back(IITDescriptor::get(IITDescriptor::Token, 0));
return;
@@ -864,6 +949,10 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
OutputTable.push_back(IITDescriptor::getVector(128, IsScalableVector));
DecodeIITType(NextElt, Infos, Info, OutputTable);
return;
+ case IIT_V256:
+ OutputTable.push_back(IITDescriptor::getVector(256, IsScalableVector));
+ DecodeIITType(NextElt, Infos, Info, OutputTable);
+ return;
case IIT_V512:
OutputTable.push_back(IITDescriptor::getVector(512, IsScalableVector));
DecodeIITType(NextElt, Infos, Info, OutputTable);
@@ -932,6 +1021,7 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
case IIT_EMPTYSTRUCT:
OutputTable.push_back(IITDescriptor::get(IITDescriptor::Struct, 0));
return;
+ case IIT_STRUCT9: ++StructElts; LLVM_FALLTHROUGH;
case IIT_STRUCT8: ++StructElts; LLVM_FALLTHROUGH;
case IIT_STRUCT7: ++StructElts; LLVM_FALLTHROUGH;
case IIT_STRUCT6: ++StructElts; LLVM_FALLTHROUGH;
@@ -1025,6 +1115,7 @@ static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos,
case IITDescriptor::Void: return Type::getVoidTy(Context);
case IITDescriptor::VarArg: return Type::getVoidTy(Context);
case IITDescriptor::MMX: return Type::getX86_MMXTy(Context);
+ case IITDescriptor::AMX: return Type::getX86_AMXTy(Context);
case IITDescriptor::Token: return Type::getTokenTy(Context);
case IITDescriptor::Metadata: return Type::getMetadataTy(Context);
case IITDescriptor::Half: return Type::getHalfTy(Context);
@@ -1162,7 +1253,7 @@ Function *Intrinsic::getDeclaration(Module *M, ID id, ArrayRef<Type*> Tys) {
// There can never be multiple globals with the same name of different types,
// because intrinsics must be a specific type.
return cast<Function>(
- M->getOrInsertFunction(getName(id, Tys),
+ M->getOrInsertFunction(Tys.empty() ? getName(id) : getName(id, Tys),
getType(M->getContext(), id, Tys))
.getCallee());
}
@@ -1204,6 +1295,7 @@ static bool matchIntrinsicType(
case IITDescriptor::Void: return !Ty->isVoidTy();
case IITDescriptor::VarArg: return true;
case IITDescriptor::MMX: return !Ty->isX86_MMXTy();
+ case IITDescriptor::AMX: return !Ty->isX86_AMXTy();
case IITDescriptor::Token: return !Ty->isTokenTy();
case IITDescriptor::Metadata: return !Ty->isMetadataTy();
case IITDescriptor::Half: return !Ty->isHalfTy();
@@ -1356,10 +1448,10 @@ static bool matchIntrinsicType(
// Verify the overloaded type "matches" the Ref type.
// i.e. Ty is a vector with the same width as Ref.
// Composed of pointers to the same element type as Ref.
- VectorType *ReferenceType = dyn_cast<VectorType>(ArgTys[RefArgNumber]);
- VectorType *ThisArgVecTy = dyn_cast<VectorType>(Ty);
+ auto *ReferenceType = dyn_cast<VectorType>(ArgTys[RefArgNumber]);
+ auto *ThisArgVecTy = dyn_cast<VectorType>(Ty);
if (!ThisArgVecTy || !ReferenceType ||
- (ReferenceType->getNumElements() != ThisArgVecTy->getNumElements()))
+ (ReferenceType->getElementCount() != ThisArgVecTy->getElementCount()))
return true;
PointerType *ThisArgEltTy =
dyn_cast<PointerType>(ThisArgVecTy->getElementType());
diff --git a/llvm/lib/IR/Globals.cpp b/llvm/lib/IR/Globals.cpp
index dd8e62164de1..c2cbe7ddddf8 100644
--- a/llvm/lib/IR/Globals.cpp
+++ b/llvm/lib/IR/Globals.cpp
@@ -104,7 +104,8 @@ bool GlobalValue::isInterposable() const {
bool GlobalValue::canBenefitFromLocalAlias() const {
// See AsmPrinter::getSymbolPreferLocal().
- return GlobalObject::isExternalLinkage(getLinkage()) && !isDeclaration() &&
+ return hasDefaultVisibility() &&
+ GlobalObject::isExternalLinkage(getLinkage()) && !isDeclaration() &&
!isa<GlobalIFunc>(this) && !hasComdat();
}
@@ -351,11 +352,15 @@ GlobalVariable::GlobalVariable(Type *Ty, bool constant, LinkageTypes Link,
GlobalVariable::GlobalVariable(Module &M, Type *Ty, bool constant,
LinkageTypes Link, Constant *InitVal,
const Twine &Name, GlobalVariable *Before,
- ThreadLocalMode TLMode, unsigned AddressSpace,
+ ThreadLocalMode TLMode,
+ Optional<unsigned> AddressSpace,
bool isExternallyInitialized)
: GlobalObject(Ty, Value::GlobalVariableVal,
OperandTraits<GlobalVariable>::op_begin(this),
- InitVal != nullptr, Link, Name, AddressSpace),
+ InitVal != nullptr, Link, Name,
+ AddressSpace
+ ? *AddressSpace
+ : M.getDataLayout().getDefaultGlobalsAddressSpace()),
isConstantGlobal(constant),
isExternallyInitializedConstant(isExternallyInitialized) {
assert(!Ty->isFunctionTy() && PointerType::isValidElementType(Ty) &&
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index b87dfe1c8df6..91ca984b755c 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -42,13 +42,14 @@ using namespace llvm;
/// created.
GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
const Twine &Name,
- unsigned AddressSpace) {
+ unsigned AddressSpace,
+ Module *M) {
Constant *StrConstant = ConstantDataArray::getString(Context, Str);
- Module &M = *BB->getParent()->getParent();
- auto *GV = new GlobalVariable(M, StrConstant->getType(), true,
- GlobalValue::PrivateLinkage, StrConstant, Name,
- nullptr, GlobalVariable::NotThreadLocal,
- AddressSpace);
+ if (!M)
+ M = BB->getParent()->getParent();
+ auto *GV = new GlobalVariable(
+ *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
+ StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
GV->setAlignment(Align(1));
return GV;
@@ -79,6 +80,17 @@ static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
return CI;
}
+Value *IRBuilderBase::CreateVScale(Constant *Scaling, const Twine &Name) {
+ Module *M = GetInsertBlock()->getParent()->getParent();
+ assert(isa<ConstantInt>(Scaling) && "Expected constant integer");
+ Function *TheFn =
+ Intrinsic::getDeclaration(M, Intrinsic::vscale, {Scaling->getType()});
+ CallInst *CI = createCallHelper(TheFn, {}, this, Name);
+ return cast<ConstantInt>(Scaling)->getSExtValue() == 1
+ ? CI
+ : CreateMul(CI, Scaling);
+}
+
CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
MaybeAlign Align, bool isVolatile,
MDNode *TBAATag, MDNode *ScopeTag,
@@ -135,22 +147,21 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
return CI;
}
-CallInst *IRBuilderBase::CreateMemCpy(Value *Dst, MaybeAlign DstAlign,
- Value *Src, MaybeAlign SrcAlign,
- Value *Size, bool isVolatile,
- MDNode *TBAATag, MDNode *TBAAStructTag,
- MDNode *ScopeTag, MDNode *NoAliasTag) {
+CallInst *IRBuilderBase::CreateMemTransferInst(
+ Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
+ MaybeAlign SrcAlign, Value *Size, bool isVolatile, MDNode *TBAATag,
+ MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
Dst = getCastedInt8PtrValue(Dst);
Src = getCastedInt8PtrValue(Src);
Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
Module *M = BB->getParent()->getParent();
- Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
+ Function *TheFn = Intrinsic::getDeclaration(M, IntrID, Tys);
CallInst *CI = createCallHelper(TheFn, Ops, this);
- auto* MCI = cast<MemCpyInst>(CI);
+ auto* MCI = cast<MemTransferInst>(CI);
if (DstAlign)
MCI->setDestAlignment(*DstAlign);
if (SrcAlign)
@@ -324,78 +335,57 @@ static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
Module *M = GetInsertBlock()->getParent()->getParent();
Value *Ops[] = {Acc, Src};
- Type *Tys[] = {Acc->getType(), Src->getType()};
- auto Decl = Intrinsic::getDeclaration(
- M, Intrinsic::experimental_vector_reduce_v2_fadd, Tys);
+ auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fadd,
+ {Src->getType()});
return createCallHelper(Decl, Ops, this);
}
CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
Module *M = GetInsertBlock()->getParent()->getParent();
Value *Ops[] = {Acc, Src};
- Type *Tys[] = {Acc->getType(), Src->getType()};
- auto Decl = Intrinsic::getDeclaration(
- M, Intrinsic::experimental_vector_reduce_v2_fmul, Tys);
+ auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fmul,
+ {Src->getType()});
return createCallHelper(Decl, Ops, this);
}
CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
- return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_add,
- Src);
+ return getReductionIntrinsic(this, Intrinsic::vector_reduce_add, Src);
}
CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
- return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_mul,
- Src);
+ return getReductionIntrinsic(this, Intrinsic::vector_reduce_mul, Src);
}
CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
- return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_and,
- Src);
+ return getReductionIntrinsic(this, Intrinsic::vector_reduce_and, Src);
}
CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
- return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_or,
- Src);
+ return getReductionIntrinsic(this, Intrinsic::vector_reduce_or, Src);
}
CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
- return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_xor,
- Src);
+ return getReductionIntrinsic(this, Intrinsic::vector_reduce_xor, Src);
}
CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
- auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smax
- : Intrinsic::experimental_vector_reduce_umax;
+ auto ID =
+ IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
return getReductionIntrinsic(this, ID, Src);
}
CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
- auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smin
- : Intrinsic::experimental_vector_reduce_umin;
+ auto ID =
+ IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
return getReductionIntrinsic(this, ID, Src);
}
-CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) {
- auto Rdx = getReductionIntrinsic(
- this, Intrinsic::experimental_vector_reduce_fmax, Src);
- if (NoNaN) {
- FastMathFlags FMF;
- FMF.setNoNaNs();
- Rdx->setFastMathFlags(FMF);
- }
- return Rdx;
+CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src) {
+ return getReductionIntrinsic(this, Intrinsic::vector_reduce_fmax, Src);
}
-CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) {
- auto Rdx = getReductionIntrinsic(
- this, Intrinsic::experimental_vector_reduce_fmin, Src);
- if (NoNaN) {
- FastMathFlags FMF;
- FMF.setNoNaNs();
- Rdx->setFastMathFlags(FMF);
- }
- return Rdx;
+CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src) {
+ return getReductionIntrinsic(this, Intrinsic::vector_reduce_fmin, Src);
}
CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
@@ -462,6 +452,13 @@ IRBuilderBase::CreateAssumption(Value *Cond,
return createCallHelper(FnAssume, Ops, this, "", nullptr, OpBundles);
}
+Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
+ Module *M = BB->getModule();
+ auto *FnIntrinsic = Intrinsic::getDeclaration(
+ M, Intrinsic::experimental_noalias_scope_decl, {});
+ return createCallHelper(FnIntrinsic, {Scope}, this);
+}
+
/// Create a call to a Masked Load intrinsic.
/// \p Ptr - base pointer for the load
/// \p Alignment - alignment of the source location
@@ -525,8 +522,8 @@ CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, Align Alignment,
Value *Mask, Value *PassThru,
const Twine &Name) {
- auto PtrsTy = cast<VectorType>(Ptrs->getType());
- auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
+ auto *PtrsTy = cast<FixedVectorType>(Ptrs->getType());
+ auto *PtrTy = cast<PointerType>(PtrsTy->getElementType());
unsigned NumElts = PtrsTy->getNumElements();
auto *DataTy = FixedVectorType::get(PtrTy->getElementType(), NumElts);
@@ -555,8 +552,8 @@ CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, Align Alignment,
/// be accessed in memory
CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
Align Alignment, Value *Mask) {
- auto PtrsTy = cast<VectorType>(Ptrs->getType());
- auto DataTy = cast<VectorType>(Data->getType());
+ auto *PtrsTy = cast<FixedVectorType>(Ptrs->getType());
+ auto *DataTy = cast<FixedVectorType>(Data->getType());
unsigned NumElts = PtrsTy->getNumElements();
#ifndef NDEBUG
@@ -588,7 +585,7 @@ getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
Args.push_back(ActualCallee);
Args.push_back(B.getInt32(CallArgs.size()));
Args.push_back(B.getInt32(Flags));
- Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
+ llvm::append_range(Args, CallArgs);
// GC Transition and Deopt args are now always handled via operand bundle.
// They will be removed from the signature of gc.statepoint shortly.
Args.push_back(B.getInt32(0));
@@ -605,18 +602,17 @@ getStatepointBundles(Optional<ArrayRef<T1>> TransitionArgs,
std::vector<OperandBundleDef> Rval;
if (DeoptArgs) {
SmallVector<Value*, 16> DeoptValues;
- DeoptValues.insert(DeoptValues.end(), DeoptArgs->begin(), DeoptArgs->end());
+ llvm::append_range(DeoptValues, *DeoptArgs);
Rval.emplace_back("deopt", DeoptValues);
}
if (TransitionArgs) {
SmallVector<Value*, 16> TransitionValues;
- TransitionValues.insert(TransitionValues.end(),
- TransitionArgs->begin(), TransitionArgs->end());
+ llvm::append_range(TransitionValues, *TransitionArgs);
Rval.emplace_back("gc-transition", TransitionValues);
}
if (GCArgs.size()) {
SmallVector<Value*, 16> LiveValues;
- LiveValues.insert(LiveValues.end(), GCArgs.begin(), GCArgs.end());
+ llvm::append_range(LiveValues, GCArgs);
Rval.emplace_back("gc-live", LiveValues);
}
return Rval;
@@ -662,10 +658,10 @@ CallInst *IRBuilderBase::CreateGCStatepointCall(
CallInst *IRBuilderBase::CreateGCStatepointCall(
uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
- ArrayRef<Use> CallArgs, Optional<ArrayRef<Use>> TransitionArgs,
+ ArrayRef<Value *> CallArgs, Optional<ArrayRef<Use>> TransitionArgs,
Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
const Twine &Name) {
- return CreateGCStatepointCallCommon<Use, Use, Use, Value *>(
+ return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>(
this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
DeoptArgs, GCArgs, Name);
}
@@ -720,9 +716,9 @@ InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
- ArrayRef<Use> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs,
+ ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs,
Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
- return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>(
+ return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>(
this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
}
@@ -999,18 +995,24 @@ Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
const Twine &Name) {
- assert(NumElts > 0 && "Cannot splat to an empty vector!");
+ auto EC = ElementCount::getFixed(NumElts);
+ return CreateVectorSplat(EC, V, Name);
+}
+
+Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
+ const Twine &Name) {
+ assert(EC.isNonZero() && "Cannot splat to an empty vector!");
- // First insert it into an undef vector so we can shuffle it.
+ // First insert it into a poison vector so we can shuffle it.
Type *I32Ty = getInt32Ty();
- Value *Undef = UndefValue::get(FixedVectorType::get(V->getType(), NumElts));
- V = CreateInsertElement(Undef, V, ConstantInt::get(I32Ty, 0),
+ Value *Poison = PoisonValue::get(VectorType::get(V->getType(), EC));
+ V = CreateInsertElement(Poison, V, ConstantInt::get(I32Ty, 0),
Name + ".splatinsert");
// Shuffle the value across the desired number of elements.
- Value *Zeros =
- ConstantAggregateZero::get(FixedVectorType::get(I32Ty, NumElts));
- return CreateShuffleVector(V, Undef, Zeros, Name + ".splat");
+ SmallVector<int, 16> Zeros;
+ Zeros.resize(EC.getKnownMinValue());
+ return CreateShuffleVector(V, Zeros, Name + ".splat");
}
Value *IRBuilderBase::CreateExtractInteger(
@@ -1045,9 +1047,7 @@ Value *IRBuilderBase::CreatePreserveArrayAccessIndex(
Value *LastIndexV = getInt32(LastIndex);
Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
- SmallVector<Value *, 4> IdxList;
- for (unsigned I = 0; I < Dimension; ++I)
- IdxList.push_back(Zero);
+ SmallVector<Value *, 4> IdxList(Dimension, Zero);
IdxList.push_back(LastIndexV);
Type *ResultType =
diff --git a/llvm/lib/IR/IRPrintingPasses.cpp b/llvm/lib/IR/IRPrintingPasses.cpp
index 03657ff8d9d4..8d6fe1eb6134 100644
--- a/llvm/lib/IR/IRPrintingPasses.cpp
+++ b/llvm/lib/IR/IRPrintingPasses.cpp
@@ -11,13 +11,15 @@
//===----------------------------------------------------------------------===//
#include "llvm/IR/IRPrintingPasses.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Module.h"
-#include "llvm/IR/PassManager.h"
+#include "llvm/IR/PrintPasses.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
+
using namespace llvm;
PrintModulePass::PrintModulePass() : OS(dbgs()) {}
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index bfbd801cb7a7..1e3fcd672a43 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -55,9 +55,6 @@ Instruction::~Instruction() {
// instructions in a BasicBlock are deleted).
if (isUsedByMetadata())
ValueAsMetadata::handleRAUW(this, UndefValue::get(getType()));
-
- if (hasMetadataHashEntry())
- clearMetadataHashEntries();
}
@@ -488,6 +485,7 @@ bool Instruction::isIdenticalToWhenDefined(const Instruction *I) const {
if (!std::equal(op_begin(), op_end(), I->op_begin()))
return false;
+ // WARNING: this logic must be kept in sync with EliminateDuplicatePHINodes()!
if (const PHINode *thisPHI = dyn_cast<PHINode>(this)) {
const PHINode *otherPHI = cast<PHINode>(I);
return std::equal(thisPHI->block_begin(), thisPHI->block_end(),
@@ -643,16 +641,18 @@ bool Instruction::isLifetimeStartOrEnd() const {
return ID == Intrinsic::lifetime_start || ID == Intrinsic::lifetime_end;
}
-const Instruction *Instruction::getNextNonDebugInstruction() const {
+const Instruction *
+Instruction::getNextNonDebugInstruction(bool SkipPseudoOp) const {
for (const Instruction *I = getNextNode(); I; I = I->getNextNode())
- if (!isa<DbgInfoIntrinsic>(I))
+ if (!isa<DbgInfoIntrinsic>(I) && !(SkipPseudoOp && isa<PseudoProbeInst>(I)))
return I;
return nullptr;
}
-const Instruction *Instruction::getPrevNonDebugInstruction() const {
+const Instruction *
+Instruction::getPrevNonDebugInstruction(bool SkipPseudoOp) const {
for (const Instruction *I = getPrevNode(); I; I = I->getPrevNode())
- if (!isa<DbgInfoIntrinsic>(I))
+ if (!isa<DbgInfoIntrinsic>(I) && !(SkipPseudoOp && isa<PseudoProbeInst>(I)))
return I;
return nullptr;
}
@@ -672,6 +672,13 @@ bool Instruction::isAssociative() const {
}
}
+bool Instruction::isCommutative() const {
+ if (auto *II = dyn_cast<IntrinsicInst>(this))
+ return II->isCommutative();
+ // TODO: Should allow icmp/fcmp?
+ return isCommutative(getOpcode());
+}
+
unsigned Instruction::getNumSuccessors() const {
switch (getOpcode()) {
#define HANDLE_TERM_INST(N, OPC, CLASS) \
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 2f17a0d73af4..d6b4a4f5030f 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -49,13 +49,14 @@ using namespace llvm;
// AllocaInst Class
//===----------------------------------------------------------------------===//
-Optional<uint64_t>
+Optional<TypeSize>
AllocaInst::getAllocationSizeInBits(const DataLayout &DL) const {
- uint64_t Size = DL.getTypeAllocSizeInBits(getAllocatedType());
+ TypeSize Size = DL.getTypeAllocSizeInBits(getAllocatedType());
if (isArrayAllocation()) {
auto *C = dyn_cast<ConstantInt>(getArraySize());
if (!C)
return None;
+ assert(!Size.isScalable() && "Array elements cannot have a scalable size");
Size *= C->getZExtValue();
}
return Size;
@@ -321,16 +322,6 @@ Value *CallBase::getReturnedArgOperand() const {
return nullptr;
}
-bool CallBase::hasRetAttr(Attribute::AttrKind Kind) const {
- if (Attrs.hasAttribute(AttributeList::ReturnIndex, Kind))
- return true;
-
- // Look at the callee, if available.
- if (const Function *F = getCalledFunction())
- return F->getAttributes().hasAttribute(AttributeList::ReturnIndex, Kind);
- return false;
-}
-
/// Determine whether the argument or parameter has the given attribute.
bool CallBase::paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const {
assert(ArgNo < getNumArgOperands() && "Param index out of bounds!");
@@ -409,7 +400,7 @@ CallBase::BundleOpInfo &CallBase::getBundleOpInfoForOperand(unsigned OpIdx) {
bundle_op_iterator Begin = bundle_op_info_begin();
bundle_op_iterator End = bundle_op_info_end();
- bundle_op_iterator Current;
+ bundle_op_iterator Current = Begin;
while (Begin != End) {
unsigned ScaledOperandPerBundle =
@@ -515,6 +506,18 @@ CallInst *CallInst::Create(CallInst *CI, ArrayRef<OperandBundleDef> OpB,
return NewCI;
}
+CallInst *CallInst::CreateWithReplacedBundle(CallInst *CI, OperandBundleDef OpB,
+ Instruction *InsertPt) {
+ SmallVector<OperandBundleDef, 2> OpDefs;
+ for (unsigned i = 0, e = CI->getNumOperandBundles(); i < e; ++i) {
+ auto ChildOB = CI->getOperandBundleAt(i);
+ if (ChildOB.getTagName() != OpB.getTag())
+ OpDefs.emplace_back(ChildOB);
+ }
+ OpDefs.emplace_back(OpB);
+ return CallInst::Create(CI, OpDefs, InsertPt);
+}
+
// Update profile weight for call instruction by scaling it using the ratio
// of S/T. The meaning of "branch_weights" meta data for call instruction is
// transfered to represent call count.
@@ -548,8 +551,9 @@ void CallInst::updateProfWeight(uint64_t S, uint64_t T) {
->getValue()
.getZExtValue());
Val *= APS;
- Vals.push_back(MDB.createConstant(ConstantInt::get(
- Type::getInt64Ty(getContext()), Val.udiv(APT).getLimitedValue())));
+ Vals.push_back(MDB.createConstant(
+ ConstantInt::get(Type::getInt32Ty(getContext()),
+ Val.udiv(APT).getLimitedValue(UINT32_MAX))));
} else if (ProfDataName->getString().equals("VP"))
for (unsigned i = 1; i < ProfileData->getNumOperands(); i += 2) {
// The first value is the key of the value profile, which will not change.
@@ -826,6 +830,18 @@ InvokeInst *InvokeInst::Create(InvokeInst *II, ArrayRef<OperandBundleDef> OpB,
return NewII;
}
+InvokeInst *InvokeInst::CreateWithReplacedBundle(InvokeInst *II,
+ OperandBundleDef OpB,
+ Instruction *InsertPt) {
+ SmallVector<OperandBundleDef, 2> OpDefs;
+ for (unsigned i = 0, e = II->getNumOperandBundles(); i < e; ++i) {
+ auto ChildOB = II->getOperandBundleAt(i);
+ if (ChildOB.getTagName() != OpB.getTag())
+ OpDefs.emplace_back(ChildOB);
+ }
+ OpDefs.emplace_back(OpB);
+ return InvokeInst::Create(II, OpDefs, InsertPt);
+}
LandingPadInst *InvokeInst::getLandingPadInst() const {
return cast<LandingPadInst>(getUnwindDest()->getFirstNonPHI());
@@ -1919,7 +1935,7 @@ ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
}
void ShuffleVectorInst::commute() {
- int NumOpElts = cast<VectorType>(Op<0>()->getType())->getNumElements();
+ int NumOpElts = cast<FixedVectorType>(Op<0>()->getType())->getNumElements();
int NumMaskElts = ShuffleMask.size();
SmallVector<int, 16> NewMask(NumMaskElts);
for (int i = 0; i != NumMaskElts; ++i) {
@@ -1943,7 +1959,8 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
return false;
// Make sure the mask elements make sense.
- int V1Size = cast<VectorType>(V1->getType())->getElementCount().Min;
+ int V1Size =
+ cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
for (int Elem : Mask)
if (Elem != UndefMaskElem && Elem >= V1Size * 2)
return false;
@@ -1973,7 +1990,7 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
return true;
if (const auto *MV = dyn_cast<ConstantVector>(Mask)) {
- unsigned V1Size = cast<VectorType>(V1->getType())->getNumElements();
+ unsigned V1Size = cast<FixedVectorType>(V1->getType())->getNumElements();
for (Value *Op : MV->operands()) {
if (auto *CI = dyn_cast<ConstantInt>(Op)) {
if (CI->uge(V1Size*2))
@@ -1986,8 +2003,9 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
}
if (const auto *CDS = dyn_cast<ConstantDataSequential>(Mask)) {
- unsigned V1Size = cast<VectorType>(V1->getType())->getNumElements();
- for (unsigned i = 0, e = MaskTy->getNumElements(); i != e; ++i)
+ unsigned V1Size = cast<FixedVectorType>(V1->getType())->getNumElements();
+ for (unsigned i = 0, e = cast<FixedVectorType>(MaskTy)->getNumElements();
+ i != e; ++i)
if (CDS->getElementAsInteger(i) >= V1Size*2)
return false;
return true;
@@ -1998,12 +2016,26 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
void ShuffleVectorInst::getShuffleMask(const Constant *Mask,
SmallVectorImpl<int> &Result) {
- unsigned NumElts = cast<VectorType>(Mask->getType())->getElementCount().Min;
+ ElementCount EC = cast<VectorType>(Mask->getType())->getElementCount();
+
if (isa<ConstantAggregateZero>(Mask)) {
- Result.resize(NumElts, 0);
+ Result.resize(EC.getKnownMinValue(), 0);
return;
}
- Result.reserve(NumElts);
+
+ Result.reserve(EC.getKnownMinValue());
+
+ if (EC.isScalable()) {
+ assert((isa<ConstantAggregateZero>(Mask) || isa<UndefValue>(Mask)) &&
+ "Scalable vector shuffle mask must be undef or zeroinitializer");
+ int MaskVal = isa<UndefValue>(Mask) ? -1 : 0;
+ for (unsigned I = 0; I < EC.getKnownMinValue(); ++I)
+ Result.emplace_back(MaskVal);
+ return;
+ }
+
+ unsigned NumElts = EC.getKnownMinValue();
+
if (auto *CDS = dyn_cast<ConstantDataSequential>(Mask)) {
for (unsigned i = 0; i != NumElts; ++i)
Result.push_back(CDS->getElementAsInteger(i));
@@ -2185,8 +2217,14 @@ bool ShuffleVectorInst::isExtractSubvectorMask(ArrayRef<int> Mask,
bool ShuffleVectorInst::isIdentityWithPadding() const {
if (isa<UndefValue>(Op<2>()))
return false;
- int NumOpElts = cast<VectorType>(Op<0>()->getType())->getNumElements();
- int NumMaskElts = cast<VectorType>(getType())->getNumElements();
+
+ // FIXME: Not currently possible to express a shuffle mask for a scalable
+ // vector for this case.
+ if (isa<ScalableVectorType>(getType()))
+ return false;
+
+ int NumOpElts = cast<FixedVectorType>(Op<0>()->getType())->getNumElements();
+ int NumMaskElts = cast<FixedVectorType>(getType())->getNumElements();
if (NumMaskElts <= NumOpElts)
return false;
@@ -2208,7 +2246,7 @@ bool ShuffleVectorInst::isIdentityWithExtract() const {
return false;
// FIXME: Not currently possible to express a shuffle mask for a scalable
- // vector for this case
+ // vector for this case.
if (isa<ScalableVectorType>(getType()))
return false;
@@ -2226,8 +2264,13 @@ bool ShuffleVectorInst::isConcat() const {
isa<UndefValue>(Op<2>()))
return false;
- int NumOpElts = cast<VectorType>(Op<0>()->getType())->getNumElements();
- int NumMaskElts = getType()->getNumElements();
+ // FIXME: Not currently possible to express a shuffle mask for a scalable
+ // vector for this case.
+ if (isa<ScalableVectorType>(getType()))
+ return false;
+
+ int NumOpElts = cast<FixedVectorType>(Op<0>()->getType())->getNumElements();
+ int NumMaskElts = cast<FixedVectorType>(getType())->getNumElements();
if (NumMaskElts != NumOpElts * 2)
return false;
@@ -2614,6 +2657,7 @@ bool CastInst::isNoopCast(Instruction::CastOps Opcode,
Type *SrcTy,
Type *DestTy,
const DataLayout &DL) {
+ assert(castIsValid(Opcode, SrcTy, DestTy) && "method precondition");
switch (Opcode) {
default: llvm_unreachable("Invalid CastOp");
case Instruction::Trunc:
@@ -2968,8 +3012,8 @@ CastInst *CastInst::CreatePointerCast(Value *S, Type *Ty,
"Invalid cast");
assert(Ty->isVectorTy() == S->getType()->isVectorTy() && "Invalid cast");
assert((!Ty->isVectorTy() ||
- cast<VectorType>(Ty)->getNumElements() ==
- cast<VectorType>(S->getType())->getNumElements()) &&
+ cast<VectorType>(Ty)->getElementCount() ==
+ cast<VectorType>(S->getType())->getElementCount()) &&
"Invalid cast");
if (Ty->isIntOrIntVectorTy())
@@ -2987,8 +3031,8 @@ CastInst *CastInst::CreatePointerCast(Value *S, Type *Ty,
"Invalid cast");
assert(Ty->isVectorTy() == S->getType()->isVectorTy() && "Invalid cast");
assert((!Ty->isVectorTy() ||
- cast<VectorType>(Ty)->getNumElements() ==
- cast<VectorType>(S->getType())->getNumElements()) &&
+ cast<VectorType>(Ty)->getElementCount() ==
+ cast<VectorType>(S->getType())->getElementCount()) &&
"Invalid cast");
if (Ty->isIntOrIntVectorTy())
@@ -3088,63 +3132,6 @@ CastInst *CastInst::CreateFPCast(Value *C, Type *Ty,
return Create(opcode, C, Ty, Name, InsertAtEnd);
}
-// Check whether it is valid to call getCastOpcode for these types.
-// This routine must be kept in sync with getCastOpcode.
-bool CastInst::isCastable(Type *SrcTy, Type *DestTy) {
- if (!SrcTy->isFirstClassType() || !DestTy->isFirstClassType())
- return false;
-
- if (SrcTy == DestTy)
- return true;
-
- if (VectorType *SrcVecTy = dyn_cast<VectorType>(SrcTy))
- if (VectorType *DestVecTy = dyn_cast<VectorType>(DestTy))
- if (SrcVecTy->getNumElements() == DestVecTy->getNumElements()) {
- // An element by element cast. Valid if casting the elements is valid.
- SrcTy = SrcVecTy->getElementType();
- DestTy = DestVecTy->getElementType();
- }
-
- // Get the bit sizes, we'll need these
- TypeSize SrcBits = SrcTy->getPrimitiveSizeInBits(); // 0 for ptr
- TypeSize DestBits = DestTy->getPrimitiveSizeInBits(); // 0 for ptr
-
- // Run through the possibilities ...
- if (DestTy->isIntegerTy()) { // Casting to integral
- if (SrcTy->isIntegerTy()) // Casting from integral
- return true;
- if (SrcTy->isFloatingPointTy()) // Casting from floating pt
- return true;
- if (SrcTy->isVectorTy()) // Casting from vector
- return DestBits == SrcBits;
- // Casting from something else
- return SrcTy->isPointerTy();
- }
- if (DestTy->isFloatingPointTy()) { // Casting to floating pt
- if (SrcTy->isIntegerTy()) // Casting from integral
- return true;
- if (SrcTy->isFloatingPointTy()) // Casting from floating pt
- return true;
- if (SrcTy->isVectorTy()) // Casting from vector
- return DestBits == SrcBits;
- // Casting from something else
- return false;
- }
- if (DestTy->isVectorTy()) // Casting to vector
- return DestBits == SrcBits;
- if (DestTy->isPointerTy()) { // Casting to pointer
- if (SrcTy->isPointerTy()) // Casting from pointer
- return true;
- return SrcTy->isIntegerTy(); // Casting from integral
- }
- if (DestTy->isX86_MMXTy()) {
- if (SrcTy->isVectorTy())
- return DestBits == SrcBits; // 64-bit vector to MMX
- return false;
- } // Casting to something else
- return false;
-}
-
bool CastInst::isBitCastable(Type *SrcTy, Type *DestTy) {
if (!SrcTy->isFirstClassType() || !DestTy->isFirstClassType())
return false;
@@ -3206,7 +3193,6 @@ bool CastInst::isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy,
// castIsValid( getCastOpcode(Val, Ty), Val, Ty)
// should not assert in castIsValid. In other words, this produces a "correct"
// casting opcode for the arguments passed to it.
-// This routine must be kept in sync with isCastable.
Instruction::CastOps
CastInst::getCastOpcode(
const Value *Src, bool SrcIsSigned, Type *DestTy, bool DestIsSigned) {
@@ -3221,7 +3207,7 @@ CastInst::getCastOpcode(
// FIXME: Check address space sizes here
if (VectorType *SrcVecTy = dyn_cast<VectorType>(SrcTy))
if (VectorType *DestVecTy = dyn_cast<VectorType>(DestTy))
- if (SrcVecTy->getNumElements() == DestVecTy->getNumElements()) {
+ if (SrcVecTy->getElementCount() == DestVecTy->getElementCount()) {
// An element by element cast. Find the appropriate opcode based on the
// element types.
SrcTy = SrcVecTy->getElementType();
@@ -3311,10 +3297,7 @@ CastInst::getCastOpcode(
/// it in one place and to eliminate the redundant code for getting the sizes
/// of the types involved.
bool
-CastInst::castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) {
- // Check for type sanity on the arguments
- Type *SrcTy = S->getType();
-
+CastInst::castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy) {
if (!SrcTy->isFirstClassType() || !DstTy->isFirstClassType() ||
SrcTy->isAggregateType() || DstTy->isAggregateType())
return false;
@@ -3330,9 +3313,9 @@ CastInst::castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) {
// scalar types means that checking that vector lengths match also checks that
// scalars are not being converted to vectors or vectors to scalars).
ElementCount SrcEC = SrcIsVec ? cast<VectorType>(SrcTy)->getElementCount()
- : ElementCount(0, false);
+ : ElementCount::getFixed(0);
ElementCount DstEC = DstIsVec ? cast<VectorType>(DstTy)->getElementCount()
- : ElementCount(0, false);
+ : ElementCount::getFixed(0);
// Switch on the opcode provided
switch (op) {
@@ -3390,9 +3373,9 @@ CastInst::castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) {
if (SrcIsVec && DstIsVec)
return SrcEC == DstEC;
if (SrcIsVec)
- return SrcEC == ElementCount(1, false);
+ return SrcEC == ElementCount::getFixed(1);
if (DstIsVec)
- return DstEC == ElementCount(1, false);
+ return DstEC == ElementCount::getFixed(1);
return true;
}
@@ -3643,10 +3626,12 @@ bool CmpInst::isCommutative() const {
return cast<FCmpInst>(this)->isCommutative();
}
-bool CmpInst::isEquality() const {
- if (const ICmpInst *IC = dyn_cast<ICmpInst>(this))
- return IC->isEquality();
- return cast<FCmpInst>(this)->isEquality();
+bool CmpInst::isEquality(Predicate P) {
+ if (ICmpInst::isIntPredicate(P))
+ return ICmpInst::isEquality(P);
+ if (FCmpInst::isFPPredicate(P))
+ return FCmpInst::isEquality(P);
+ llvm_unreachable("Unsupported predicate kind");
}
CmpInst::Predicate CmpInst::getInversePredicate(Predicate pred) {
@@ -3740,29 +3725,6 @@ ICmpInst::Predicate ICmpInst::getUnsignedPredicate(Predicate pred) {
}
}
-CmpInst::Predicate CmpInst::getFlippedStrictnessPredicate(Predicate pred) {
- switch (pred) {
- default: llvm_unreachable("Unknown or unsupported cmp predicate!");
- case ICMP_SGT: return ICMP_SGE;
- case ICMP_SLT: return ICMP_SLE;
- case ICMP_SGE: return ICMP_SGT;
- case ICMP_SLE: return ICMP_SLT;
- case ICMP_UGT: return ICMP_UGE;
- case ICMP_ULT: return ICMP_ULE;
- case ICMP_UGE: return ICMP_UGT;
- case ICMP_ULE: return ICMP_ULT;
-
- case FCMP_OGT: return FCMP_OGE;
- case FCMP_OLT: return FCMP_OLE;
- case FCMP_OGE: return FCMP_OGT;
- case FCMP_OLE: return FCMP_OLT;
- case FCMP_UGT: return FCMP_UGE;
- case FCMP_ULT: return FCMP_ULE;
- case FCMP_UGE: return FCMP_UGT;
- case FCMP_ULE: return FCMP_ULT;
- }
-}
-
CmpInst::Predicate CmpInst::getSwappedPredicate(Predicate pred) {
switch (pred) {
default: llvm_unreachable("Unknown cmp predicate!");
@@ -3793,22 +3755,97 @@ CmpInst::Predicate CmpInst::getSwappedPredicate(Predicate pred) {
}
}
+bool CmpInst::isNonStrictPredicate(Predicate pred) {
+ switch (pred) {
+ case ICMP_SGE:
+ case ICMP_SLE:
+ case ICMP_UGE:
+ case ICMP_ULE:
+ case FCMP_OGE:
+ case FCMP_OLE:
+ case FCMP_UGE:
+ case FCMP_ULE:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool CmpInst::isStrictPredicate(Predicate pred) {
+ switch (pred) {
+ case ICMP_SGT:
+ case ICMP_SLT:
+ case ICMP_UGT:
+ case ICMP_ULT:
+ case FCMP_OGT:
+ case FCMP_OLT:
+ case FCMP_UGT:
+ case FCMP_ULT:
+ return true;
+ default:
+ return false;
+ }
+}
+
+CmpInst::Predicate CmpInst::getStrictPredicate(Predicate pred) {
+ switch (pred) {
+ case ICMP_SGE:
+ return ICMP_SGT;
+ case ICMP_SLE:
+ return ICMP_SLT;
+ case ICMP_UGE:
+ return ICMP_UGT;
+ case ICMP_ULE:
+ return ICMP_ULT;
+ case FCMP_OGE:
+ return FCMP_OGT;
+ case FCMP_OLE:
+ return FCMP_OLT;
+ case FCMP_UGE:
+ return FCMP_UGT;
+ case FCMP_ULE:
+ return FCMP_ULT;
+ default:
+ return pred;
+ }
+}
+
CmpInst::Predicate CmpInst::getNonStrictPredicate(Predicate pred) {
switch (pred) {
- case ICMP_SGT: return ICMP_SGE;
- case ICMP_SLT: return ICMP_SLE;
- case ICMP_UGT: return ICMP_UGE;
- case ICMP_ULT: return ICMP_ULE;
- case FCMP_OGT: return FCMP_OGE;
- case FCMP_OLT: return FCMP_OLE;
- case FCMP_UGT: return FCMP_UGE;
- case FCMP_ULT: return FCMP_ULE;
- default: return pred;
+ case ICMP_SGT:
+ return ICMP_SGE;
+ case ICMP_SLT:
+ return ICMP_SLE;
+ case ICMP_UGT:
+ return ICMP_UGE;
+ case ICMP_ULT:
+ return ICMP_ULE;
+ case FCMP_OGT:
+ return FCMP_OGE;
+ case FCMP_OLT:
+ return FCMP_OLE;
+ case FCMP_UGT:
+ return FCMP_UGE;
+ case FCMP_ULT:
+ return FCMP_ULE;
+ default:
+ return pred;
}
}
+CmpInst::Predicate CmpInst::getFlippedStrictnessPredicate(Predicate pred) {
+ assert(CmpInst::isRelational(pred) && "Call only with relational predicate!");
+
+ if (isStrictPredicate(pred))
+ return getNonStrictPredicate(pred);
+ if (isNonStrictPredicate(pred))
+ return getStrictPredicate(pred);
+
+ llvm_unreachable("Unknown predicate!");
+}
+
CmpInst::Predicate CmpInst::getSignedPredicate(Predicate pred) {
- assert(CmpInst::isUnsigned(pred) && "Call only with signed predicates!");
+ assert(CmpInst::isUnsigned(pred) && "Call only with unsigned predicates!");
switch (pred) {
default:
@@ -3824,6 +3861,23 @@ CmpInst::Predicate CmpInst::getSignedPredicate(Predicate pred) {
}
}
+CmpInst::Predicate CmpInst::getUnsignedPredicate(Predicate pred) {
+ assert(CmpInst::isSigned(pred) && "Call only with signed predicates!");
+
+ switch (pred) {
+ default:
+ llvm_unreachable("Unknown predicate!");
+ case CmpInst::ICMP_SLT:
+ return CmpInst::ICMP_ULT;
+ case CmpInst::ICMP_SLE:
+ return CmpInst::ICMP_ULE;
+ case CmpInst::ICMP_SGT:
+ return CmpInst::ICMP_UGT;
+ case CmpInst::ICMP_SGE:
+ return CmpInst::ICMP_UGE;
+ }
+}
+
bool CmpInst::isUnsigned(Predicate predicate) {
switch (predicate) {
default: return false;
@@ -3840,6 +3894,18 @@ bool CmpInst::isSigned(Predicate predicate) {
}
}
+CmpInst::Predicate CmpInst::getFlippedSignednessPredicate(Predicate pred) {
+ assert(CmpInst::isRelational(pred) &&
+ "Call only with non-equality predicates!");
+
+ if (isSigned(pred))
+ return getUnsignedPredicate(pred);
+ if (isUnsigned(pred))
+ return getSignedPredicate(pred);
+
+ llvm_unreachable("Unknown predicate!");
+}
+
bool CmpInst::isOrdered(Predicate predicate) {
switch (predicate) {
default: return false;
diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp
index c4e06cd979ed..3d1ea2853591 100644
--- a/llvm/lib/IR/IntrinsicInst.cpp
+++ b/llvm/lib/IR/IntrinsicInst.cpp
@@ -208,7 +208,7 @@ Optional<int> VPIntrinsic::GetMaskParamPos(Intrinsic::ID IntrinsicID) {
default:
return None;
-#define REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
+#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
case Intrinsic::VPID: \
return MASKPOS;
#include "llvm/IR/VPIntrinsics.def"
@@ -220,7 +220,7 @@ Optional<int> VPIntrinsic::GetVectorLengthParamPos(Intrinsic::ID IntrinsicID) {
default:
return None;
-#define REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
+#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
case Intrinsic::VPID: \
return VLENPOS;
#include "llvm/IR/VPIntrinsics.def"
@@ -232,7 +232,7 @@ bool VPIntrinsic::IsVPIntrinsic(Intrinsic::ID ID) {
default:
return false;
-#define REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
+#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
case Intrinsic::VPID: \
break;
#include "llvm/IR/VPIntrinsics.def"
@@ -242,25 +242,26 @@ bool VPIntrinsic::IsVPIntrinsic(Intrinsic::ID ID) {
// Equivalent non-predicated opcode
unsigned VPIntrinsic::GetFunctionalOpcodeForVP(Intrinsic::ID ID) {
+ unsigned FunctionalOC = Instruction::Call;
switch (ID) {
default:
- return Instruction::Call;
-
-#define HANDLE_VP_TO_OC(VPID, OC) \
- case Intrinsic::VPID: \
- return Instruction::OC;
+ break;
+#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
+#define HANDLE_VP_TO_OPC(OPC) FunctionalOC = Instruction::OPC;
+#define END_REGISTER_VP_INTRINSIC(...) break;
#include "llvm/IR/VPIntrinsics.def"
}
+
+ return FunctionalOC;
}
-Intrinsic::ID VPIntrinsic::GetForOpcode(unsigned OC) {
- switch (OC) {
+Intrinsic::ID VPIntrinsic::GetForOpcode(unsigned IROPC) {
+ switch (IROPC) {
default:
return Intrinsic::not_intrinsic;
-#define HANDLE_VP_TO_OC(VPID, OC) \
- case Instruction::OC: \
- return Intrinsic::VPID;
+#define HANDLE_VP_TO_OPC(OPC) case Instruction::OPC:
+#define END_REGISTER_VP_INTRINSIC(VPID) return Intrinsic::VPID;
#include "llvm/IR/VPIntrinsics.def"
}
}
@@ -280,8 +281,8 @@ bool VPIntrinsic::canIgnoreVectorLengthParam() const {
// the operation. This function returns true when this is detected statically
// in the IR.
- // Check whether "W == vscale * EC.Min"
- if (EC.Scalable) {
+ // Check whether "W == vscale * EC.getKnownMinValue()"
+ if (EC.isScalable()) {
// Undig the DL
auto ParMod = this->getModule();
if (!ParMod)
@@ -291,8 +292,8 @@ bool VPIntrinsic::canIgnoreVectorLengthParam() const {
// Compare vscale patterns
uint64_t VScaleFactor;
if (match(VLParam, m_c_Mul(m_ConstantInt(VScaleFactor), m_VScale(DL))))
- return VScaleFactor >= EC.Min;
- return (EC.Min == 1) && match(VLParam, m_VScale(DL));
+ return VScaleFactor >= EC.getKnownMinValue();
+ return (EC.getKnownMinValue() == 1) && match(VLParam, m_VScale(DL));
}
// standard SIMD operation
@@ -301,7 +302,7 @@ bool VPIntrinsic::canIgnoreVectorLengthParam() const {
return false;
uint64_t VLNum = VLConst->getZExtValue();
- if (VLNum >= EC.Min)
+ if (VLNum >= EC.getKnownMinValue())
return true;
return false;
diff --git a/llvm/lib/IR/LLVMContext.cpp b/llvm/lib/IR/LLVMContext.cpp
index 7ebca5274369..4f292101256d 100644
--- a/llvm/lib/IR/LLVMContext.cpp
+++ b/llvm/lib/IR/LLVMContext.cpp
@@ -146,11 +146,16 @@ bool LLVMContext::getDiagnosticsHotnessRequested() const {
return pImpl->DiagnosticsHotnessRequested;
}
-void LLVMContext::setDiagnosticsHotnessThreshold(uint64_t Threshold) {
+void LLVMContext::setDiagnosticsHotnessThreshold(Optional<uint64_t> Threshold) {
pImpl->DiagnosticsHotnessThreshold = Threshold;
}
+
uint64_t LLVMContext::getDiagnosticsHotnessThreshold() const {
- return pImpl->DiagnosticsHotnessThreshold;
+ return pImpl->DiagnosticsHotnessThreshold.getValueOr(UINT64_MAX);
+}
+
+bool LLVMContext::isDiagnosticsHotnessThresholdSetFromPSI() const {
+ return !pImpl->DiagnosticsHotnessThreshold.hasValue();
}
remarks::RemarkStreamer *LLVMContext::getMainRemarkStreamer() {
diff --git a/llvm/lib/IR/LLVMContextImpl.cpp b/llvm/lib/IR/LLVMContextImpl.cpp
index f197b3e67d30..e998138ec3cb 100644
--- a/llvm/lib/IR/LLVMContextImpl.cpp
+++ b/llvm/lib/IR/LLVMContextImpl.cpp
@@ -35,6 +35,7 @@ LLVMContextImpl::LLVMContextImpl(LLVMContext &C)
FP128Ty(C, Type::FP128TyID),
PPC_FP128Ty(C, Type::PPC_FP128TyID),
X86_MMXTy(C, Type::X86_MMXTyID),
+ X86_AMXTy(C, Type::X86_AMXTyID),
Int1Ty(C, 1),
Int8Ty(C, 8),
Int16Ty(C, 16),
@@ -50,11 +51,10 @@ LLVMContextImpl::~LLVMContextImpl() {
delete *OwnedModules.begin();
#ifndef NDEBUG
- // Check for metadata references from leaked Instructions.
- for (auto &Pair : InstructionMetadata)
+ // Check for metadata references from leaked Values.
+ for (auto &Pair : ValueMetadata)
Pair.first->dump();
- assert(InstructionMetadata.empty() &&
- "Instructions with metadata have been leaked");
+ assert(ValueMetadata.empty() && "Values with metadata have been leaked");
#endif
// Drop references for MDNodes. Do this before Values get deleted to avoid
@@ -98,11 +98,9 @@ LLVMContextImpl::~LLVMContextImpl() {
CAZConstants.clear();
CPNConstants.clear();
UVConstants.clear();
+ PVConstants.clear();
IntConstants.clear();
FPConstants.clear();
-
- for (auto &CDSConstant : CDSConstants)
- delete CDSConstant.second;
CDSConstants.clear();
// Destroy attribute node lists.
@@ -129,8 +127,15 @@ LLVMContextImpl::~LLVMContextImpl() {
}
void LLVMContextImpl::dropTriviallyDeadConstantArrays() {
- SmallSetVector<ConstantArray *, 4> WorkList(ArrayConstants.begin(),
- ArrayConstants.end());
+ SmallSetVector<ConstantArray *, 4> WorkList;
+
+ // When ArrayConstants are of substantial size and only a few in them are
+ // dead, starting WorkList with all elements of ArrayConstants can be
+ // wasteful. Instead, starting WorkList with only elements that have empty
+ // uses.
+ for (ConstantArray *C : ArrayConstants)
+ if (C->use_empty())
+ WorkList.insert(C);
while (!WorkList.empty()) {
ConstantArray *C = WorkList.pop_back_val();
@@ -171,7 +176,7 @@ unsigned MDNodeOpsKey::calculateHash(MDNode *N, unsigned Offset) {
unsigned Hash = hash_combine_range(N->op_begin() + Offset, N->op_end());
#ifndef NDEBUG
{
- SmallVector<Metadata *, 8> MDs(N->op_begin() + Offset, N->op_end());
+ SmallVector<Metadata *, 8> MDs(drop_begin(N->operands(), Offset));
unsigned RawHash = calculateHash(MDs);
assert(Hash == RawHash &&
"Expected hash of MDOperand to equal hash of Metadata*");
@@ -215,19 +220,8 @@ void LLVMContextImpl::getSyncScopeNames(
SSNs[SSE.second] = SSE.first();
}
-/// Singleton instance of the OptBisect class.
-///
-/// This singleton is accessed via the LLVMContext::getOptPassGate() function.
-/// It provides a mechanism to disable passes and individual optimizations at
-/// compile time based on a command line option (-opt-bisect-limit) in order to
-/// perform a bisecting search for optimization-related problems.
-///
-/// Even if multiple LLVMContext objects are created, they will all return the
-/// same instance of OptBisect in order to provide a single bisect count. Any
-/// code that uses the OptBisect object should be serialized when bisection is
-/// enabled in order to enable a consistent bisect count.
-static ManagedStatic<OptBisect> OptBisector;
-
+/// Gets the OptPassGate for this LLVMContextImpl, which defaults to the
+/// singleton OptBisect if not explicitly set.
OptPassGate &LLVMContextImpl::getOptPassGate() const {
if (!OPG)
OPG = &(*OptBisector);
diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h
index 1c7d8746d242..05fd1814e2dc 100644
--- a/llvm/lib/IR/LLVMContextImpl.h
+++ b/llvm/lib/IR/LLVMContextImpl.h
@@ -57,27 +57,7 @@ class Type;
class Value;
class ValueHandleBase;
-struct DenseMapAPIntKeyInfo {
- static inline APInt getEmptyKey() {
- APInt V(nullptr, 0);
- V.U.VAL = 0;
- return V;
- }
-
- static inline APInt getTombstoneKey() {
- APInt V(nullptr, 0);
- V.U.VAL = 1;
- return V;
- }
-
- static unsigned getHashValue(const APInt &Key) {
- return static_cast<unsigned>(hash_value(Key));
- }
-
- static bool isEqual(const APInt &LHS, const APInt &RHS) {
- return LHS.getBitWidth() == RHS.getBitWidth() && LHS == RHS;
- }
-};
+using DenseMapAPIntKeyInfo = DenseMapInfo<APInt>;
struct DenseMapAPFloatKeyInfo {
static inline APFloat getEmptyKey() { return APFloat(APFloat::Bogus(), 1); }
@@ -366,6 +346,36 @@ template <> struct MDNodeKeyImpl<DISubrange> {
}
};
+template <> struct MDNodeKeyImpl<DIGenericSubrange> {
+ Metadata *CountNode;
+ Metadata *LowerBound;
+ Metadata *UpperBound;
+ Metadata *Stride;
+
+ MDNodeKeyImpl(Metadata *CountNode, Metadata *LowerBound, Metadata *UpperBound,
+ Metadata *Stride)
+ : CountNode(CountNode), LowerBound(LowerBound), UpperBound(UpperBound),
+ Stride(Stride) {}
+ MDNodeKeyImpl(const DIGenericSubrange *N)
+ : CountNode(N->getRawCountNode()), LowerBound(N->getRawLowerBound()),
+ UpperBound(N->getRawUpperBound()), Stride(N->getRawStride()) {}
+
+ bool isKeyOf(const DIGenericSubrange *RHS) const {
+ return (CountNode == RHS->getRawCountNode()) &&
+ (LowerBound == RHS->getRawLowerBound()) &&
+ (UpperBound == RHS->getRawUpperBound()) &&
+ (Stride == RHS->getRawStride());
+ }
+
+ unsigned getHashValue() const {
+ auto *MD = dyn_cast_or_null<ConstantAsMetadata>(CountNode);
+ if (CountNode && MD)
+ return hash_combine(cast<ConstantInt>(MD->getValue())->getSExtValue(),
+ LowerBound, UpperBound, Stride);
+ return hash_combine(CountNode, LowerBound, UpperBound, Stride);
+ }
+};
+
template <> struct MDNodeKeyImpl<DIEnumerator> {
APInt Value;
MDString *Name;
@@ -417,6 +427,37 @@ template <> struct MDNodeKeyImpl<DIBasicType> {
}
};
+template <> struct MDNodeKeyImpl<DIStringType> {
+ unsigned Tag;
+ MDString *Name;
+ Metadata *StringLength;
+ Metadata *StringLengthExp;
+ uint64_t SizeInBits;
+ uint32_t AlignInBits;
+ unsigned Encoding;
+
+ MDNodeKeyImpl(unsigned Tag, MDString *Name, Metadata *StringLength,
+ Metadata *StringLengthExp, uint64_t SizeInBits,
+ uint32_t AlignInBits, unsigned Encoding)
+ : Tag(Tag), Name(Name), StringLength(StringLength),
+ StringLengthExp(StringLengthExp), SizeInBits(SizeInBits),
+ AlignInBits(AlignInBits), Encoding(Encoding) {}
+ MDNodeKeyImpl(const DIStringType *N)
+ : Tag(N->getTag()), Name(N->getRawName()),
+ StringLength(N->getRawStringLength()),
+ StringLengthExp(N->getRawStringLengthExp()),
+ SizeInBits(N->getSizeInBits()), AlignInBits(N->getAlignInBits()),
+ Encoding(N->getEncoding()) {}
+
+ bool isKeyOf(const DIStringType *RHS) const {
+ return Tag == RHS->getTag() && Name == RHS->getRawName() &&
+ SizeInBits == RHS->getSizeInBits() &&
+ AlignInBits == RHS->getAlignInBits() &&
+ Encoding == RHS->getEncoding();
+ }
+ unsigned getHashValue() const { return hash_combine(Tag, Name, Encoding); }
+};
+
template <> struct MDNodeKeyImpl<DIDerivedType> {
unsigned Tag;
MDString *Name;
@@ -525,6 +566,9 @@ template <> struct MDNodeKeyImpl<DICompositeType> {
MDString *Identifier;
Metadata *Discriminator;
Metadata *DataLocation;
+ Metadata *Associated;
+ Metadata *Allocated;
+ Metadata *Rank;
MDNodeKeyImpl(unsigned Tag, MDString *Name, Metadata *File, unsigned Line,
Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits,
@@ -532,13 +576,15 @@ template <> struct MDNodeKeyImpl<DICompositeType> {
Metadata *Elements, unsigned RuntimeLang,
Metadata *VTableHolder, Metadata *TemplateParams,
MDString *Identifier, Metadata *Discriminator,
- Metadata *DataLocation)
+ Metadata *DataLocation, Metadata *Associated,
+ Metadata *Allocated, Metadata *Rank)
: Tag(Tag), Name(Name), File(File), Line(Line), Scope(Scope),
BaseType(BaseType), SizeInBits(SizeInBits), OffsetInBits(OffsetInBits),
AlignInBits(AlignInBits), Flags(Flags), Elements(Elements),
RuntimeLang(RuntimeLang), VTableHolder(VTableHolder),
TemplateParams(TemplateParams), Identifier(Identifier),
- Discriminator(Discriminator), DataLocation(DataLocation) {}
+ Discriminator(Discriminator), DataLocation(DataLocation),
+ Associated(Associated), Allocated(Allocated), Rank(Rank) {}
MDNodeKeyImpl(const DICompositeType *N)
: Tag(N->getTag()), Name(N->getRawName()), File(N->getRawFile()),
Line(N->getLine()), Scope(N->getRawScope()),
@@ -549,7 +595,9 @@ template <> struct MDNodeKeyImpl<DICompositeType> {
TemplateParams(N->getRawTemplateParams()),
Identifier(N->getRawIdentifier()),
Discriminator(N->getRawDiscriminator()),
- DataLocation(N->getRawDataLocation()) {}
+ DataLocation(N->getRawDataLocation()),
+ Associated(N->getRawAssociated()), Allocated(N->getRawAllocated()),
+ Rank(N->getRawRank()) {}
bool isKeyOf(const DICompositeType *RHS) const {
return Tag == RHS->getTag() && Name == RHS->getRawName() &&
@@ -564,7 +612,9 @@ template <> struct MDNodeKeyImpl<DICompositeType> {
TemplateParams == RHS->getRawTemplateParams() &&
Identifier == RHS->getRawIdentifier() &&
Discriminator == RHS->getRawDiscriminator() &&
- DataLocation == RHS->getRawDataLocation();
+ DataLocation == RHS->getRawDataLocation() &&
+ Associated == RHS->getRawAssociated() &&
+ Allocated == RHS->getRawAllocated() && Rank == RHS->getRawRank();
}
unsigned getHashValue() const {
@@ -841,25 +891,28 @@ template <> struct MDNodeKeyImpl<DIModule> {
MDString *IncludePath;
MDString *APINotesFile;
unsigned LineNo;
+ bool IsDecl;
MDNodeKeyImpl(Metadata *File, Metadata *Scope, MDString *Name,
MDString *ConfigurationMacros, MDString *IncludePath,
- MDString *APINotesFile, unsigned LineNo)
+ MDString *APINotesFile, unsigned LineNo, bool IsDecl)
: File(File), Scope(Scope), Name(Name),
ConfigurationMacros(ConfigurationMacros), IncludePath(IncludePath),
- APINotesFile(APINotesFile), LineNo(LineNo) {}
+ APINotesFile(APINotesFile), LineNo(LineNo), IsDecl(IsDecl) {}
MDNodeKeyImpl(const DIModule *N)
: File(N->getRawFile()), Scope(N->getRawScope()), Name(N->getRawName()),
ConfigurationMacros(N->getRawConfigurationMacros()),
IncludePath(N->getRawIncludePath()),
- APINotesFile(N->getRawAPINotesFile()), LineNo(N->getLineNo()) {}
+ APINotesFile(N->getRawAPINotesFile()), LineNo(N->getLineNo()),
+ IsDecl(N->getIsDecl()) {}
bool isKeyOf(const DIModule *RHS) const {
return Scope == RHS->getRawScope() && Name == RHS->getRawName() &&
ConfigurationMacros == RHS->getRawConfigurationMacros() &&
IncludePath == RHS->getRawIncludePath() &&
APINotesFile == RHS->getRawAPINotesFile() &&
- File == RHS->getRawFile() && LineNo == RHS->getLineNo();
+ File == RHS->getRawFile() && LineNo == RHS->getLineNo() &&
+ IsDecl == RHS->getIsDecl();
}
unsigned getHashValue() const {
@@ -1204,72 +1257,55 @@ template <class NodeTy> struct MDNodeInfo {
#define HANDLE_MDNODE_LEAF(CLASS) using CLASS##Info = MDNodeInfo<CLASS>;
#include "llvm/IR/Metadata.def"
-/// Map-like storage for metadata attachments.
-class MDAttachmentMap {
- SmallVector<std::pair<unsigned, TrackingMDNodeRef>, 2> Attachments;
-
+/// Multimap-like storage for metadata attachments.
+class MDAttachments {
public:
- bool empty() const { return Attachments.empty(); }
- size_t size() const { return Attachments.size(); }
-
- /// Get a particular attachment (if any).
- MDNode *lookup(unsigned ID) const;
-
- /// Set an attachment to a particular node.
- ///
- /// Set the \c ID attachment to \c MD, replacing the current attachment at \c
- /// ID (if anyway).
- void set(unsigned ID, MDNode &MD);
-
- /// Remove an attachment.
- ///
- /// Remove the attachment at \c ID, if any.
- bool erase(unsigned ID);
-
- /// Copy out all the attachments.
- ///
- /// Copies all the current attachments into \c Result, sorting by attachment
- /// ID. This function does \em not clear \c Result.
- void getAll(SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const;
-
- /// Erase matching attachments.
- ///
- /// Erases all attachments matching the \c shouldRemove predicate.
- template <class PredTy> void remove_if(PredTy shouldRemove) {
- Attachments.erase(llvm::remove_if(Attachments, shouldRemove),
- Attachments.end());
- }
-};
-
-/// Multimap-like storage for metadata attachments for globals. This differs
-/// from MDAttachmentMap in that it allows multiple attachments per metadata
-/// kind.
-class MDGlobalAttachmentMap {
struct Attachment {
unsigned MDKind;
TrackingMDNodeRef Node;
};
+
+private:
SmallVector<Attachment, 1> Attachments;
public:
bool empty() const { return Attachments.empty(); }
-
- /// Appends all attachments with the given ID to \c Result in insertion order.
- /// If the global has no attachments with the given ID, or if ID is invalid,
- /// leaves Result unchanged.
- void get(unsigned ID, SmallVectorImpl<MDNode *> &Result) const;
+ size_t size() const { return Attachments.size(); }
/// Returns the first attachment with the given ID or nullptr if no such
/// attachment exists.
MDNode *lookup(unsigned ID) const;
- void insert(unsigned ID, MDNode &MD);
- bool erase(unsigned ID);
+ /// Appends all attachments with the given ID to \c Result in insertion order.
+ /// If the global has no attachments with the given ID, or if ID is invalid,
+ /// leaves Result unchanged.
+ void get(unsigned ID, SmallVectorImpl<MDNode *> &Result) const;
/// Appends all attachments for the global to \c Result, sorting by attachment
/// ID. Attachments with the same ID appear in insertion order. This function
/// does \em not clear \c Result.
void getAll(SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const;
+
+ /// Set an attachment to a particular node.
+ ///
+ /// Set the \c ID attachment to \c MD, replacing the current attachments at \c
+ /// ID (if anyway).
+ void set(unsigned ID, MDNode *MD);
+
+ /// Adds an attachment to a particular node.
+ void insert(unsigned ID, MDNode &MD);
+
+ /// Remove attachments with the given ID.
+ ///
+ /// Remove the attachments at \c ID, if any.
+ bool erase(unsigned ID);
+
+ /// Erase matching attachments.
+ ///
+ /// Erases all attachments matching the \c shouldRemove predicate.
+ template <class PredTy> void remove_if(PredTy shouldRemove) {
+ llvm::erase_if(Attachments, shouldRemove);
+ }
};
class LLVMContextImpl {
@@ -1289,7 +1325,26 @@ public:
std::unique_ptr<DiagnosticHandler> DiagHandler;
bool RespectDiagnosticFilters = false;
bool DiagnosticsHotnessRequested = false;
- uint64_t DiagnosticsHotnessThreshold = 0;
+ /// The minimum hotness value a diagnostic needs in order to be included in
+ /// optimization diagnostics.
+ ///
+ /// The threshold is an Optional value, which maps to one of the 3 states:
+ /// 1). 0 => threshold disabled. All emarks will be printed.
+ /// 2). positive int => manual threshold by user. Remarks with hotness exceed
+ /// threshold will be printed.
+ /// 3). None => 'auto' threshold by user. The actual value is not
+ /// available at command line, but will be synced with
+ /// hotness threhold from profile summary during
+ /// compilation.
+ ///
+ /// State 1 and 2 are considered as terminal states. State transition is
+ /// only allowed from 3 to 2, when the threshold is first synced with profile
+ /// summary. This ensures that the threshold is set only once and stays
+ /// constant.
+ ///
+ /// If threshold option is not specified, it is disabled (0) by default.
+ Optional<uint64_t> DiagnosticsHotnessThreshold = 0;
+
/// The specialized remark streamer used by LLVM's OptimizationRemarkEmitter.
std::unique_ptr<LLVMRemarkStreamer> LLVMRS;
@@ -1342,10 +1397,15 @@ public:
DenseMap<Type *, std::unique_ptr<UndefValue>> UVConstants;
- StringMap<ConstantDataSequential*> CDSConstants;
+ DenseMap<Type *, std::unique_ptr<PoisonValue>> PVConstants;
+
+ StringMap<std::unique_ptr<ConstantDataSequential>> CDSConstants;
DenseMap<std::pair<const Function *, const BasicBlock *>, BlockAddress *>
BlockAddresses;
+
+ DenseMap<const GlobalValue *, DSOLocalEquivalent *> DSOLocalEquivalents;
+
ConstantUniqueMap<ConstantExpr> ExprConstants;
ConstantUniqueMap<InlineAsm> InlineAsms;
@@ -1358,7 +1418,7 @@ public:
// Basic type instances.
Type VoidTy, LabelTy, HalfTy, BFloatTy, FloatTy, DoubleTy, MetadataTy,
TokenTy;
- Type X86_FP80Ty, FP128Ty, PPC_FP128Ty, X86_MMXTy;
+ Type X86_FP80Ty, FP128Ty, PPC_FP128Ty, X86_MMXTy, X86_AMXTy;
IntegerType Int1Ty, Int8Ty, Int16Ty, Int32Ty, Int64Ty, Int128Ty;
BumpPtrAllocator Alloc;
@@ -1387,11 +1447,8 @@ public:
/// CustomMDKindNames - Map to hold the metadata string to ID mapping.
StringMap<unsigned> CustomMDKindNames;
- /// Collection of per-instruction metadata used in this context.
- DenseMap<const Instruction *, MDAttachmentMap> InstructionMetadata;
-
- /// Collection of per-GlobalObject metadata used in this context.
- DenseMap<const GlobalObject *, MDGlobalAttachmentMap> GlobalObjectMetadata;
+ /// Collection of metadata used in this context.
+ DenseMap<const Value *, MDAttachments> ValueMetadata;
/// Collection of per-GlobalObject sections used in this context.
DenseMap<const GlobalObject *, StringRef> GlobalObjectSections;
diff --git a/llvm/lib/IR/LLVMRemarkStreamer.cpp b/llvm/lib/IR/LLVMRemarkStreamer.cpp
index 96001ab42c38..18b47611c97c 100644
--- a/llvm/lib/IR/LLVMRemarkStreamer.cpp
+++ b/llvm/lib/IR/LLVMRemarkStreamer.cpp
@@ -92,12 +92,11 @@ char LLVMRemarkSetupFormatError::ID = 0;
Expected<std::unique_ptr<ToolOutputFile>> llvm::setupLLVMOptimizationRemarks(
LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses,
StringRef RemarksFormat, bool RemarksWithHotness,
- unsigned RemarksHotnessThreshold) {
+ Optional<uint64_t> RemarksHotnessThreshold) {
if (RemarksWithHotness)
Context.setDiagnosticsHotnessRequested(true);
- if (RemarksHotnessThreshold)
- Context.setDiagnosticsHotnessThreshold(RemarksHotnessThreshold);
+ Context.setDiagnosticsHotnessThreshold(RemarksHotnessThreshold);
if (RemarksFilename.empty())
return nullptr;
@@ -137,16 +136,14 @@ Expected<std::unique_ptr<ToolOutputFile>> llvm::setupLLVMOptimizationRemarks(
return std::move(RemarksFile);
}
-Error llvm::setupLLVMOptimizationRemarks(LLVMContext &Context, raw_ostream &OS,
- StringRef RemarksPasses,
- StringRef RemarksFormat,
- bool RemarksWithHotness,
- unsigned RemarksHotnessThreshold) {
+Error llvm::setupLLVMOptimizationRemarks(
+ LLVMContext &Context, raw_ostream &OS, StringRef RemarksPasses,
+ StringRef RemarksFormat, bool RemarksWithHotness,
+ Optional<uint64_t> RemarksHotnessThreshold) {
if (RemarksWithHotness)
Context.setDiagnosticsHotnessRequested(true);
- if (RemarksHotnessThreshold)
- Context.setDiagnosticsHotnessThreshold(RemarksHotnessThreshold);
+ Context.setDiagnosticsHotnessThreshold(RemarksHotnessThreshold);
Expected<remarks::Format> Format = remarks::parseFormat(RemarksFormat);
if (Error E = Format.takeError())
diff --git a/llvm/lib/IR/LegacyPassManager.cpp b/llvm/lib/IR/LegacyPassManager.cpp
index 74869fa62c66..4547c3a01239 100644
--- a/llvm/lib/IR/LegacyPassManager.cpp
+++ b/llvm/lib/IR/LegacyPassManager.cpp
@@ -20,6 +20,8 @@
#include "llvm/IR/LegacyPassNameParser.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PassTimingInfo.h"
+#include "llvm/IR/PrintPasses.h"
+#include "llvm/IR/StructuralHash.h"
#include "llvm/Support/Chrono.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
@@ -48,7 +50,7 @@ namespace {
enum PassDebugLevel {
Disabled, Arguments, Structure, Executions, Details
};
-}
+} // namespace
static cl::opt<enum PassDebugLevel>
PassDebugging("debug-pass", cl::Hidden,
@@ -60,80 +62,6 @@ PassDebugging("debug-pass", cl::Hidden,
clEnumVal(Executions, "print pass name before it is executed"),
clEnumVal(Details , "print pass details when it is executed")));
-namespace {
-typedef llvm::cl::list<const llvm::PassInfo *, bool, PassNameParser>
-PassOptionList;
-}
-
-// Print IR out before/after specified passes.
-static PassOptionList
-PrintBefore("print-before",
- llvm::cl::desc("Print IR before specified passes"),
- cl::Hidden);
-
-static PassOptionList
-PrintAfter("print-after",
- llvm::cl::desc("Print IR after specified passes"),
- cl::Hidden);
-
-static cl::opt<bool> PrintBeforeAll("print-before-all",
- llvm::cl::desc("Print IR before each pass"),
- cl::init(false), cl::Hidden);
-static cl::opt<bool> PrintAfterAll("print-after-all",
- llvm::cl::desc("Print IR after each pass"),
- cl::init(false), cl::Hidden);
-
-static cl::opt<bool>
- PrintModuleScope("print-module-scope",
- cl::desc("When printing IR for print-[before|after]{-all} "
- "always print a module IR"),
- cl::init(false), cl::Hidden);
-
-static cl::list<std::string>
- PrintFuncsList("filter-print-funcs", cl::value_desc("function names"),
- cl::desc("Only print IR for functions whose name "
- "match this for all print-[before|after][-all] "
- "options"),
- cl::CommaSeparated, cl::Hidden);
-
-/// This is a helper to determine whether to print IR before or
-/// after a pass.
-
-bool llvm::shouldPrintBeforePass() {
- return PrintBeforeAll || !PrintBefore.empty();
-}
-
-bool llvm::shouldPrintAfterPass() {
- return PrintAfterAll || !PrintAfter.empty();
-}
-
-static bool ShouldPrintBeforeOrAfterPass(StringRef PassID,
- PassOptionList &PassesToPrint) {
- for (auto *PassInf : PassesToPrint) {
- if (PassInf)
- if (PassInf->getPassArgument() == PassID) {
- return true;
- }
- }
- return false;
-}
-
-bool llvm::shouldPrintBeforePass(StringRef PassID) {
- return PrintBeforeAll || ShouldPrintBeforeOrAfterPass(PassID, PrintBefore);
-}
-
-bool llvm::shouldPrintAfterPass(StringRef PassID) {
- return PrintAfterAll || ShouldPrintBeforeOrAfterPass(PassID, PrintAfter);
-}
-
-bool llvm::forcePrintModuleIR() { return PrintModuleScope; }
-
-bool llvm::isFunctionInPrintList(StringRef FunctionName) {
- static std::unordered_set<std::string> PrintFuncNames(PrintFuncsList.begin(),
- PrintFuncsList.end());
- return PrintFuncNames.empty() ||
- PrintFuncNames.count(std::string(FunctionName));
-}
/// isPassDebuggingExecutionsOrMore - Return true if -debug-pass=Executions
/// or higher is specified.
bool PMDataManager::isPassDebuggingExecutionsOrMore() const {
@@ -209,8 +137,7 @@ void PMDataManager::emitInstrCountChangedRemark(
// remarks. Since it's possible that the first function in the module
// doesn't actually contain a basic block, we have to go and find one that's
// suitable for emitting remarks.
- auto It = std::find_if(M.begin(), M.end(),
- [](const Function &Fn) { return !Fn.empty(); });
+ auto It = llvm::find_if(M, [](const Function &Fn) { return !Fn.empty(); });
// Didn't find a function. Quit.
if (It == M.end())
@@ -641,7 +568,12 @@ PMTopLevelManager::setLastUser(ArrayRef<Pass*> AnalysisPasses, Pass *P) {
PDepth = P->getResolver()->getPMDataManager().getDepth();
for (Pass *AP : AnalysisPasses) {
- LastUser[AP] = P;
+ // Record P as the new last user of AP.
+ auto &LastUserOfAP = LastUser[AP];
+ if (LastUserOfAP)
+ InversedLastUser[LastUserOfAP].erase(AP);
+ LastUserOfAP = P;
+ InversedLastUser[P].insert(AP);
if (P == AP)
continue;
@@ -671,31 +603,25 @@ PMTopLevelManager::setLastUser(ArrayRef<Pass*> AnalysisPasses, Pass *P) {
if (P->getResolver())
setLastUser(LastPMUses, P->getResolver()->getPMDataManager().getAsPass());
-
// If AP is the last user of other passes then make P last user of
// such passes.
- for (auto LU : LastUser) {
- if (LU.second == AP)
- // DenseMap iterator is not invalidated here because
- // this is just updating existing entries.
- LastUser[LU.first] = P;
- }
+ auto &LastUsedByAP = InversedLastUser[AP];
+ for (Pass *L : LastUsedByAP)
+ LastUser[L] = P;
+ InversedLastUser[P].insert(LastUsedByAP.begin(), LastUsedByAP.end());
+ LastUsedByAP.clear();
}
}
/// Collect passes whose last user is P
void PMTopLevelManager::collectLastUses(SmallVectorImpl<Pass *> &LastUses,
Pass *P) {
- DenseMap<Pass *, SmallPtrSet<Pass *, 8> >::iterator DMI =
- InversedLastUser.find(P);
+ auto DMI = InversedLastUser.find(P);
if (DMI == InversedLastUser.end())
return;
- SmallPtrSet<Pass *, 8> &LU = DMI->second;
- for (Pass *LUP : LU) {
- LastUses.push_back(LUP);
- }
-
+ auto &LU = DMI->second;
+ LastUses.append(LU.begin(), LU.end());
}
AnalysisUsage *PMTopLevelManager::findAnalysisUsage(Pass *P) {
@@ -929,11 +855,6 @@ void PMTopLevelManager::initializeAllAnalysisInfo() {
// Initailize other pass managers
for (PMDataManager *IPM : IndirectPassManagers)
IPM->initializeAnalysisInfo();
-
- for (auto LU : LastUser) {
- SmallPtrSet<Pass *, 8> &L = InversedLastUser[LU.second];
- L.insert(LU.first);
- }
}
/// Destructor
@@ -1189,12 +1110,6 @@ void PMDataManager::collectRequiredAndUsedAnalyses(
UP.push_back(AnalysisPass);
else
RP_NotAvail.push_back(RequiredID);
-
- for (const auto &RequiredID : AnUsage->getRequiredTransitiveSet())
- if (Pass *AnalysisPass = findAnalysisPass(RequiredID, true))
- UP.push_back(AnalysisPass);
- else
- RP_NotAvail.push_back(RequiredID);
}
// All Required analyses should be available to the pass as it runs! Here
@@ -1236,6 +1151,8 @@ Pass *PMDataManager::findAnalysisPass(AnalysisID AID, bool SearchParent) {
// Print list of passes that are last used by P.
void PMDataManager::dumpLastUses(Pass *P, unsigned Offset) const{
+ if (PassDebugging < Details)
+ return;
SmallVector<Pass *, 12> LUses;
@@ -1391,8 +1308,8 @@ PMDataManager::~PMDataManager() {
//===----------------------------------------------------------------------===//
// NOTE: Is this the right place to define this method ?
// getAnalysisIfAvailable - Return analysis result or null if it doesn't exist.
-Pass *AnalysisResolver::getAnalysisIfAvailable(AnalysisID ID, bool dir) const {
- return PM.findAnalysisPass(ID, dir);
+Pass *AnalysisResolver::getAnalysisIfAvailable(AnalysisID ID) const {
+ return PM.findAnalysisPass(ID, true);
}
std::tuple<Pass *, bool>
@@ -1475,75 +1392,6 @@ void FPPassManager::dumpPassStructure(unsigned Offset) {
}
}
-#ifdef EXPENSIVE_CHECKS
-namespace {
-namespace details {
-
-// Basic hashing mechanism to detect structural change to the IR, used to verify
-// pass return status consistency with actual change. Loosely copied from
-// llvm/lib/Transforms/Utils/FunctionComparator.cpp
-
-class StructuralHash {
- uint64_t Hash = 0x6acaa36bef8325c5ULL;
-
- void update(uint64_t V) { Hash = hashing::detail::hash_16_bytes(Hash, V); }
-
-public:
- StructuralHash() = default;
-
- void update(Function &F) {
- if (F.empty())
- return;
-
- update(F.isVarArg());
- update(F.arg_size());
-
- SmallVector<const BasicBlock *, 8> BBs;
- SmallPtrSet<const BasicBlock *, 16> VisitedBBs;
-
- BBs.push_back(&F.getEntryBlock());
- VisitedBBs.insert(BBs[0]);
- while (!BBs.empty()) {
- const BasicBlock *BB = BBs.pop_back_val();
- update(45798); // Block header
- for (auto &Inst : *BB)
- update(Inst.getOpcode());
-
- const Instruction *Term = BB->getTerminator();
- for (unsigned i = 0, e = Term->getNumSuccessors(); i != e; ++i) {
- if (!VisitedBBs.insert(Term->getSuccessor(i)).second)
- continue;
- BBs.push_back(Term->getSuccessor(i));
- }
- }
- }
-
- void update(Module &M) {
- for (Function &F : M)
- update(F);
- }
-
- uint64_t getHash() const { return Hash; }
-};
-
-} // namespace details
-
-uint64_t StructuralHash(Function &F) {
- details::StructuralHash H;
- H.update(F);
- return H.getHash();
-}
-
-uint64_t StructuralHash(Module &M) {
- details::StructuralHash H;
- H.update(M);
- return H.getHash();
-}
-
-} // end anonymous namespace
-
-#endif
-
/// Execute all of the passes scheduled for execution by invoking
/// runOnFunction method. Keep track of whether any of the passes modifies
/// the function, and if so, return true.
@@ -1586,9 +1434,12 @@ bool FPPassManager::runOnFunction(Function &F) {
#endif
LocalChanged |= FP->runOnFunction(F);
-#ifdef EXPENSIVE_CHECKS
- assert((LocalChanged || (RefHash == StructuralHash(F))) &&
- "Pass modifies its input and doesn't report it.");
+#if defined(EXPENSIVE_CHECKS) && !defined(NDEBUG)
+ if (!LocalChanged && (RefHash != StructuralHash(F))) {
+ llvm::errs() << "Pass modifies its input and doesn't report it: "
+ << FP->getPassName() << "\n";
+ llvm_unreachable("Pass modifies its input and doesn't report it");
+ }
#endif
if (EmitICRemark) {
@@ -1614,7 +1465,8 @@ bool FPPassManager::runOnFunction(Function &F) {
dumpUsedSet(FP);
verifyPreservedAnalysis(FP);
- removeNotPreservedAnalysis(FP);
+ if (LocalChanged)
+ removeNotPreservedAnalysis(FP);
recordAvailableAnalysis(FP);
removeDeadPasses(FP, F.getName(), ON_FUNCTION_MSG);
}
@@ -1723,7 +1575,8 @@ MPPassManager::runOnModule(Module &M) {
dumpUsedSet(MP);
verifyPreservedAnalysis(MP);
- removeNotPreservedAnalysis(MP);
+ if (LocalChanged)
+ removeNotPreservedAnalysis(MP);
recordAvailableAnalysis(MP);
removeDeadPasses(MP, M.getModuleIdentifier(), ON_MODULE_MSG);
}
diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp
index 40d70f43132d..35af8490287b 100644
--- a/llvm/lib/IR/MDBuilder.cpp
+++ b/llvm/lib/IR/MDBuilder.cpp
@@ -151,24 +151,20 @@ MDNode *MDBuilder::mergeCallbackEncodings(MDNode *ExistingCallbacks,
}
MDNode *MDBuilder::createAnonymousAARoot(StringRef Name, MDNode *Extra) {
- // To ensure uniqueness the root node is self-referential.
- auto Dummy = MDNode::getTemporary(Context, None);
-
- SmallVector<Metadata *, 3> Args(1, Dummy.get());
+ SmallVector<Metadata *, 3> Args(1, nullptr);
if (Extra)
Args.push_back(Extra);
if (!Name.empty())
Args.push_back(createString(Name));
- MDNode *Root = MDNode::get(Context, Args);
+ MDNode *Root = MDNode::getDistinct(Context, Args);
// At this point we have
- // !0 = metadata !{} <- dummy
- // !1 = metadata !{metadata !0} <- root
- // Replace the dummy operand with the root node itself and delete the dummy.
+ // !0 = distinct !{null} <- root
+ // Replace the reserved operand with the root node itself.
Root->replaceOperandWith(0, Root);
// We now have
- // !1 = metadata !{metadata !1} <- self-referential root
+ // !0 = distinct !{!0} <- root
return Root;
}
@@ -310,14 +306,12 @@ MDNode *MDBuilder::createIrrLoopHeaderWeight(uint64_t Weight) {
return MDNode::get(Context, Vals);
}
-MDNode *MDBuilder::createMisExpect(uint64_t Index, uint64_t LikleyWeight,
- uint64_t UnlikleyWeight) {
- auto *IntType = Type::getInt64Ty(Context);
- Metadata *Vals[] = {
- createString("misexpect"),
- createConstant(ConstantInt::get(IntType, Index)),
- createConstant(ConstantInt::get(IntType, LikleyWeight)),
- createConstant(ConstantInt::get(IntType, UnlikleyWeight)),
- };
- return MDNode::get(Context, Vals);
+MDNode *MDBuilder::createPseudoProbeDesc(uint64_t GUID, uint64_t Hash,
+ Function *F) {
+ auto *Int64Ty = Type::getInt64Ty(Context);
+ SmallVector<Metadata *, 3> Ops(3);
+ Ops[0] = createConstant(ConstantInt::get(Int64Ty, GUID));
+ Ops[1] = createConstant(ConstantInt::get(Int64Ty, Hash));
+ Ops[2] = createString(F->getName());
+ return MDNode::get(Context, Ops);
}
diff --git a/llvm/lib/IR/Mangler.cpp b/llvm/lib/IR/Mangler.cpp
index 0d66e321c396..674ba3cdaa24 100644
--- a/llvm/lib/IR/Mangler.cpp
+++ b/llvm/lib/IR/Mangler.cpp
@@ -12,6 +12,7 @@
#include "llvm/IR/Mangler.h"
#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/Triple.h"
#include "llvm/ADT/Twine.h"
#include "llvm/IR/DataLayout.h"
@@ -100,7 +101,7 @@ static void addByteCountSuffix(raw_ostream &OS, const Function *F,
for (Function::const_arg_iterator AI = F->arg_begin(), AE = F->arg_end();
AI != AE; ++AI) {
// 'Dereference' type in case of byval or inalloca parameter attribute.
- uint64_t AllocSize = AI->hasPassPointeeByValueAttr() ?
+ uint64_t AllocSize = AI->hasPassPointeeByValueCopyAttr() ?
AI->getPassPointeeByValueCopySize(DL) :
DL.getTypeAllocSize(AI->getType());
@@ -184,6 +185,25 @@ void Mangler::getNameWithPrefix(SmallVectorImpl<char> &OutName,
getNameWithPrefix(OS, GV, CannotUsePrivateLabel);
}
+// Check if the name needs quotes to be safe for the linker to interpret.
+static bool canBeUnquotedInDirective(char C) {
+ return isAlnum(C) || C == '_' || C == '$' || C == '.' || C == '@';
+}
+
+static bool canBeUnquotedInDirective(StringRef Name) {
+ if (Name.empty())
+ return false;
+
+ // If any of the characters in the string is an unacceptable character, force
+ // quotes.
+ for (char C : Name) {
+ if (!canBeUnquotedInDirective(C))
+ return false;
+ }
+
+ return true;
+}
+
void llvm::emitLinkerFlagsForGlobalCOFF(raw_ostream &OS, const GlobalValue *GV,
const Triple &TT, Mangler &Mangler) {
if (!GV->hasDLLExportStorageClass() || GV->isDeclaration())
@@ -194,6 +214,9 @@ void llvm::emitLinkerFlagsForGlobalCOFF(raw_ostream &OS, const GlobalValue *GV,
else
OS << " -export:";
+ bool NeedQuotes = GV->hasName() && !canBeUnquotedInDirective(GV->getName());
+ if (NeedQuotes)
+ OS << "\"";
if (TT.isWindowsGNUEnvironment() || TT.isWindowsCygwinEnvironment()) {
std::string Flag;
raw_string_ostream FlagOS(Flag);
@@ -206,6 +229,8 @@ void llvm::emitLinkerFlagsForGlobalCOFF(raw_ostream &OS, const GlobalValue *GV,
} else {
Mangler.getNameWithPrefix(OS, GV, false);
}
+ if (NeedQuotes)
+ OS << "\"";
if (!GV->getValueType()->isFunctionTy()) {
if (TT.isWindowsMSVCEnvironment())
@@ -221,6 +246,11 @@ void llvm::emitLinkerFlagsForUsedCOFF(raw_ostream &OS, const GlobalValue *GV,
return;
OS << " /INCLUDE:";
+ bool NeedQuotes = GV->hasName() && !canBeUnquotedInDirective(GV->getName());
+ if (NeedQuotes)
+ OS << "\"";
M.getNameWithPrefix(OS, GV, false);
+ if (NeedQuotes)
+ OS << "\"";
}
diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp
index ce89009e86eb..7ca538995db2 100644
--- a/llvm/lib/IR/Metadata.cpp
+++ b/llvm/lib/IR/Metadata.cpp
@@ -10,6 +10,7 @@
//
//===----------------------------------------------------------------------===//
+#include "llvm/IR/Metadata.h"
#include "LLVMContextImpl.h"
#include "MetadataImpl.h"
#include "SymbolTableListTraitsImpl.h"
@@ -38,7 +39,7 @@
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Metadata.h"
+#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/TrackingMDRef.h"
#include "llvm/IR/Type.h"
@@ -640,10 +641,7 @@ void MDNode::resolveCycles() {
}
static bool hasSelfReference(MDNode *N) {
- for (Metadata *MD : N->operands())
- if (MD == N)
- return true;
- return false;
+ return llvm::is_contained(N->operands(), N);
}
MDNode *MDNode::replaceWithPermanentImpl() {
@@ -925,7 +923,32 @@ MDNode *MDNode::getMostGenericAliasScope(MDNode *A, MDNode *B) {
if (!A || !B)
return nullptr;
- return concatenate(A, B);
+ // Take the intersection of domains then union the scopes
+ // within those domains
+ SmallPtrSet<const MDNode *, 16> ADomains;
+ SmallPtrSet<const MDNode *, 16> IntersectDomains;
+ SmallSetVector<Metadata *, 4> MDs;
+ for (const MDOperand &MDOp : A->operands())
+ if (const MDNode *NAMD = dyn_cast<MDNode>(MDOp))
+ if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
+ ADomains.insert(Domain);
+
+ for (const MDOperand &MDOp : B->operands())
+ if (const MDNode *NAMD = dyn_cast<MDNode>(MDOp))
+ if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
+ if (ADomains.contains(Domain)) {
+ IntersectDomains.insert(Domain);
+ MDs.insert(MDOp);
+ }
+
+ for (const MDOperand &MDOp : A->operands())
+ if (const MDNode *NAMD = dyn_cast<MDNode>(MDOp))
+ if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
+ if (IntersectDomains.contains(Domain))
+ MDs.insert(MDOp);
+
+ return MDs.empty() ? nullptr
+ : getOrSelfReference(A->getContext(), MDs.getArrayRef());
}
MDNode *MDNode::getMostGenericFPMath(MDNode *A, MDNode *B) {
@@ -1101,87 +1124,158 @@ StringRef NamedMDNode::getName() const { return StringRef(Name); }
//===----------------------------------------------------------------------===//
// Instruction Metadata method implementations.
//
-void MDAttachmentMap::set(unsigned ID, MDNode &MD) {
- for (auto &I : Attachments)
- if (I.first == ID) {
- I.second.reset(&MD);
- return;
- }
- Attachments.emplace_back(std::piecewise_construct, std::make_tuple(ID),
- std::make_tuple(&MD));
+
+MDNode *MDAttachments::lookup(unsigned ID) const {
+ for (const auto &A : Attachments)
+ if (A.MDKind == ID)
+ return A.Node;
+ return nullptr;
+}
+
+void MDAttachments::get(unsigned ID, SmallVectorImpl<MDNode *> &Result) const {
+ for (const auto &A : Attachments)
+ if (A.MDKind == ID)
+ Result.push_back(A.Node);
}
-bool MDAttachmentMap::erase(unsigned ID) {
+void MDAttachments::getAll(
+ SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const {
+ for (const auto &A : Attachments)
+ Result.emplace_back(A.MDKind, A.Node);
+
+ // Sort the resulting array so it is stable with respect to metadata IDs. We
+ // need to preserve the original insertion order though.
+ if (Result.size() > 1)
+ llvm::stable_sort(Result, less_first());
+}
+
+void MDAttachments::set(unsigned ID, MDNode *MD) {
+ erase(ID);
+ if (MD)
+ insert(ID, *MD);
+}
+
+void MDAttachments::insert(unsigned ID, MDNode &MD) {
+ Attachments.push_back({ID, TrackingMDNodeRef(&MD)});
+}
+
+bool MDAttachments::erase(unsigned ID) {
if (empty())
return false;
- // Common case is one/last value.
- if (Attachments.back().first == ID) {
+ // Common case is one value.
+ if (Attachments.size() == 1 && Attachments.back().MDKind == ID) {
Attachments.pop_back();
return true;
}
- for (auto I = Attachments.begin(), E = std::prev(Attachments.end()); I != E;
- ++I)
- if (I->first == ID) {
- *I = std::move(Attachments.back());
- Attachments.pop_back();
- return true;
- }
+ auto OldSize = Attachments.size();
+ llvm::erase_if(Attachments,
+ [ID](const Attachment &A) { return A.MDKind == ID; });
+ return OldSize != Attachments.size();
+}
- return false;
+MDNode *Value::getMetadata(unsigned KindID) const {
+ if (!hasMetadata())
+ return nullptr;
+ const auto &Info = getContext().pImpl->ValueMetadata[this];
+ assert(!Info.empty() && "bit out of sync with hash table");
+ return Info.lookup(KindID);
}
-MDNode *MDAttachmentMap::lookup(unsigned ID) const {
- for (const auto &I : Attachments)
- if (I.first == ID)
- return I.second;
- return nullptr;
+MDNode *Value::getMetadata(StringRef Kind) const {
+ if (!hasMetadata())
+ return nullptr;
+ const auto &Info = getContext().pImpl->ValueMetadata[this];
+ assert(!Info.empty() && "bit out of sync with hash table");
+ return Info.lookup(getContext().getMDKindID(Kind));
}
-void MDAttachmentMap::getAll(
- SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const {
- Result.append(Attachments.begin(), Attachments.end());
+void Value::getMetadata(unsigned KindID, SmallVectorImpl<MDNode *> &MDs) const {
+ if (hasMetadata())
+ getContext().pImpl->ValueMetadata[this].get(KindID, MDs);
+}
- // Sort the resulting array so it is stable.
- if (Result.size() > 1)
- array_pod_sort(Result.begin(), Result.end());
+void Value::getMetadata(StringRef Kind, SmallVectorImpl<MDNode *> &MDs) const {
+ if (hasMetadata())
+ getMetadata(getContext().getMDKindID(Kind), MDs);
}
-void MDGlobalAttachmentMap::insert(unsigned ID, MDNode &MD) {
- Attachments.push_back({ID, TrackingMDNodeRef(&MD)});
+void Value::getAllMetadata(
+ SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const {
+ if (hasMetadata()) {
+ assert(getContext().pImpl->ValueMetadata.count(this) &&
+ "bit out of sync with hash table");
+ const auto &Info = getContext().pImpl->ValueMetadata.find(this)->second;
+ assert(!Info.empty() && "Shouldn't have called this");
+ Info.getAll(MDs);
+ }
}
-MDNode *MDGlobalAttachmentMap::lookup(unsigned ID) const {
- for (const auto &A : Attachments)
- if (A.MDKind == ID)
- return A.Node;
- return nullptr;
+void Value::setMetadata(unsigned KindID, MDNode *Node) {
+ assert(isa<Instruction>(this) || isa<GlobalObject>(this));
+
+ // Handle the case when we're adding/updating metadata on a value.
+ if (Node) {
+ auto &Info = getContext().pImpl->ValueMetadata[this];
+ assert(!Info.empty() == HasMetadata && "bit out of sync with hash table");
+ if (Info.empty())
+ HasMetadata = true;
+ Info.set(KindID, Node);
+ return;
+ }
+
+ // Otherwise, we're removing metadata from an instruction.
+ assert((HasMetadata == (getContext().pImpl->ValueMetadata.count(this) > 0)) &&
+ "bit out of sync with hash table");
+ if (!HasMetadata)
+ return; // Nothing to remove!
+ auto &Info = getContext().pImpl->ValueMetadata[this];
+
+ // Handle removal of an existing value.
+ Info.erase(KindID);
+ if (!Info.empty())
+ return;
+ getContext().pImpl->ValueMetadata.erase(this);
+ HasMetadata = false;
}
-void MDGlobalAttachmentMap::get(unsigned ID,
- SmallVectorImpl<MDNode *> &Result) const {
- for (const auto &A : Attachments)
- if (A.MDKind == ID)
- Result.push_back(A.Node);
+void Value::setMetadata(StringRef Kind, MDNode *Node) {
+ if (!Node && !HasMetadata)
+ return;
+ setMetadata(getContext().getMDKindID(Kind), Node);
}
-bool MDGlobalAttachmentMap::erase(unsigned ID) {
- auto I = std::remove_if(Attachments.begin(), Attachments.end(),
- [ID](const Attachment &A) { return A.MDKind == ID; });
- bool Changed = I != Attachments.end();
- Attachments.erase(I, Attachments.end());
- return Changed;
+void Value::addMetadata(unsigned KindID, MDNode &MD) {
+ assert(isa<Instruction>(this) || isa<GlobalObject>(this));
+ if (!HasMetadata)
+ HasMetadata = true;
+ getContext().pImpl->ValueMetadata[this].insert(KindID, MD);
}
-void MDGlobalAttachmentMap::getAll(
- SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const {
- for (const auto &A : Attachments)
- Result.emplace_back(A.MDKind, A.Node);
+void Value::addMetadata(StringRef Kind, MDNode &MD) {
+ addMetadata(getContext().getMDKindID(Kind), MD);
+}
- // Sort the resulting array so it is stable with respect to metadata IDs. We
- // need to preserve the original insertion order though.
- llvm::stable_sort(Result, less_first());
+bool Value::eraseMetadata(unsigned KindID) {
+ // Nothing to unset.
+ if (!HasMetadata)
+ return false;
+
+ auto &Store = getContext().pImpl->ValueMetadata[this];
+ bool Changed = Store.erase(KindID);
+ if (Store.empty())
+ clearMetadata();
+ return Changed;
+}
+
+void Value::clearMetadata() {
+ if (!HasMetadata)
+ return;
+ assert(getContext().pImpl->ValueMetadata.count(this) &&
+ "bit out of sync with hash table");
+ getContext().pImpl->ValueMetadata.erase(this);
+ HasMetadata = false;
}
void Instruction::setMetadata(StringRef Kind, MDNode *Node) {
@@ -1195,29 +1289,28 @@ MDNode *Instruction::getMetadataImpl(StringRef Kind) const {
}
void Instruction::dropUnknownNonDebugMetadata(ArrayRef<unsigned> KnownIDs) {
- if (!hasMetadataHashEntry())
+ if (!Value::hasMetadata())
return; // Nothing to remove!
- auto &InstructionMetadata = getContext().pImpl->InstructionMetadata;
-
- SmallSet<unsigned, 4> KnownSet;
- KnownSet.insert(KnownIDs.begin(), KnownIDs.end());
- if (KnownSet.empty()) {
+ if (KnownIDs.empty()) {
// Just drop our entry at the store.
- InstructionMetadata.erase(this);
- setHasMetadataHashEntry(false);
+ clearMetadata();
return;
}
- auto &Info = InstructionMetadata[this];
- Info.remove_if([&KnownSet](const std::pair<unsigned, TrackingMDNodeRef> &I) {
- return !KnownSet.count(I.first);
+ SmallSet<unsigned, 4> KnownSet;
+ KnownSet.insert(KnownIDs.begin(), KnownIDs.end());
+
+ auto &MetadataStore = getContext().pImpl->ValueMetadata;
+ auto &Info = MetadataStore[this];
+ assert(!Info.empty() && "bit out of sync with hash table");
+ Info.remove_if([&KnownSet](const MDAttachments::Attachment &I) {
+ return !KnownSet.count(I.MDKind);
});
if (Info.empty()) {
// Drop our entry at the store.
- InstructionMetadata.erase(this);
- setHasMetadataHashEntry(false);
+ clearMetadata();
}
}
@@ -1231,33 +1324,28 @@ void Instruction::setMetadata(unsigned KindID, MDNode *Node) {
return;
}
- // Handle the case when we're adding/updating metadata on an instruction.
- if (Node) {
- auto &Info = getContext().pImpl->InstructionMetadata[this];
- assert(!Info.empty() == hasMetadataHashEntry() &&
- "HasMetadata bit is wonked");
- if (Info.empty())
- setHasMetadataHashEntry(true);
- Info.set(KindID, *Node);
- return;
- }
+ Value::setMetadata(KindID, Node);
+}
- // Otherwise, we're removing metadata from an instruction.
- assert((hasMetadataHashEntry() ==
- (getContext().pImpl->InstructionMetadata.count(this) > 0)) &&
- "HasMetadata bit out of date!");
- if (!hasMetadataHashEntry())
- return; // Nothing to remove!
- auto &Info = getContext().pImpl->InstructionMetadata[this];
+void Instruction::addAnnotationMetadata(StringRef Name) {
+ MDBuilder MDB(getContext());
- // Handle removal of an existing value.
- Info.erase(KindID);
-
- if (!Info.empty())
- return;
+ auto *Existing = getMetadata(LLVMContext::MD_annotation);
+ SmallVector<Metadata *, 4> Names;
+ bool AppendName = true;
+ if (Existing) {
+ auto *Tuple = cast<MDTuple>(Existing);
+ for (auto &N : Tuple->operands()) {
+ if (cast<MDString>(N.get())->getString() == Name)
+ AppendName = false;
+ Names.push_back(N.get());
+ }
+ }
+ if (AppendName)
+ Names.push_back(MDB.createString(Name));
- getContext().pImpl->InstructionMetadata.erase(this);
- setHasMetadataHashEntry(false);
+ MDNode *MD = MDTuple::get(getContext(), Names);
+ setMetadata(LLVMContext::MD_annotation, MD);
}
void Instruction::setAAMetadata(const AAMDNodes &N) {
@@ -1271,13 +1359,7 @@ MDNode *Instruction::getMetadataImpl(unsigned KindID) const {
// Handle 'dbg' as a special case since it is not stored in the hash table.
if (KindID == LLVMContext::MD_dbg)
return DbgLoc.getAsMDNode();
-
- if (!hasMetadataHashEntry())
- return nullptr;
- auto &Info = getContext().pImpl->InstructionMetadata[this];
- assert(!Info.empty() && "bit out of sync with hash table");
-
- return Info.lookup(KindID);
+ return Value::getMetadata(KindID);
}
void Instruction::getAllMetadataImpl(
@@ -1288,27 +1370,8 @@ void Instruction::getAllMetadataImpl(
if (DbgLoc) {
Result.push_back(
std::make_pair((unsigned)LLVMContext::MD_dbg, DbgLoc.getAsMDNode()));
- if (!hasMetadataHashEntry())
- return;
}
-
- assert(hasMetadataHashEntry() &&
- getContext().pImpl->InstructionMetadata.count(this) &&
- "Shouldn't have called this");
- const auto &Info = getContext().pImpl->InstructionMetadata.find(this)->second;
- assert(!Info.empty() && "Shouldn't have called this");
- Info.getAll(Result);
-}
-
-void Instruction::getAllMetadataOtherThanDebugLocImpl(
- SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const {
- Result.clear();
- assert(hasMetadataHashEntry() &&
- getContext().pImpl->InstructionMetadata.count(this) &&
- "Shouldn't have called this");
- const auto &Info = getContext().pImpl->InstructionMetadata.find(this)->second;
- assert(!Info.empty() && "Shouldn't have called this");
- Info.getAll(Result);
+ Value::getAllMetadata(Result);
}
bool Instruction::extractProfMetadata(uint64_t &TrueVal,
@@ -1372,84 +1435,6 @@ bool Instruction::extractProfTotalWeight(uint64_t &TotalVal) const {
return false;
}
-void Instruction::clearMetadataHashEntries() {
- assert(hasMetadataHashEntry() && "Caller should check");
- getContext().pImpl->InstructionMetadata.erase(this);
- setHasMetadataHashEntry(false);
-}
-
-void GlobalObject::getMetadata(unsigned KindID,
- SmallVectorImpl<MDNode *> &MDs) const {
- if (hasMetadata())
- getContext().pImpl->GlobalObjectMetadata[this].get(KindID, MDs);
-}
-
-void GlobalObject::getMetadata(StringRef Kind,
- SmallVectorImpl<MDNode *> &MDs) const {
- if (hasMetadata())
- getMetadata(getContext().getMDKindID(Kind), MDs);
-}
-
-void GlobalObject::addMetadata(unsigned KindID, MDNode &MD) {
- if (!hasMetadata())
- setHasMetadataHashEntry(true);
-
- getContext().pImpl->GlobalObjectMetadata[this].insert(KindID, MD);
-}
-
-void GlobalObject::addMetadata(StringRef Kind, MDNode &MD) {
- addMetadata(getContext().getMDKindID(Kind), MD);
-}
-
-bool GlobalObject::eraseMetadata(unsigned KindID) {
- // Nothing to unset.
- if (!hasMetadata())
- return false;
-
- auto &Store = getContext().pImpl->GlobalObjectMetadata[this];
- bool Changed = Store.erase(KindID);
- if (Store.empty())
- clearMetadata();
- return Changed;
-}
-
-void GlobalObject::getAllMetadata(
- SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const {
- MDs.clear();
-
- if (!hasMetadata())
- return;
-
- getContext().pImpl->GlobalObjectMetadata[this].getAll(MDs);
-}
-
-void GlobalObject::clearMetadata() {
- if (!hasMetadata())
- return;
- getContext().pImpl->GlobalObjectMetadata.erase(this);
- setHasMetadataHashEntry(false);
-}
-
-void GlobalObject::setMetadata(unsigned KindID, MDNode *N) {
- eraseMetadata(KindID);
- if (N)
- addMetadata(KindID, *N);
-}
-
-void GlobalObject::setMetadata(StringRef Kind, MDNode *N) {
- setMetadata(getContext().getMDKindID(Kind), N);
-}
-
-MDNode *GlobalObject::getMetadata(unsigned KindID) const {
- if (hasMetadata())
- return getContext().pImpl->GlobalObjectMetadata[this].lookup(KindID);
- return nullptr;
-}
-
-MDNode *GlobalObject::getMetadata(StringRef Kind) const {
- return getMetadata(getContext().getMDKindID(Kind));
-}
-
void GlobalObject::copyMetadata(const GlobalObject *Other, unsigned Offset) {
SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
Other->getAllMetadata(MDs);
diff --git a/llvm/lib/IR/Module.cpp b/llvm/lib/IR/Module.cpp
index 3ea181a9b48d..b4f10e2e2d23 100644
--- a/llvm/lib/IR/Module.cpp
+++ b/llvm/lib/IR/Module.cpp
@@ -582,7 +582,7 @@ void Module::setProfileSummary(Metadata *M, ProfileSummary::Kind Kind) {
setModuleFlag(ModFlagBehavior::Error, "ProfileSummary", M);
}
-Metadata *Module::getProfileSummary(bool IsCS) {
+Metadata *Module::getProfileSummary(bool IsCS) const {
return (IsCS ? getModuleFlag("CSProfileSummary")
: getModuleFlag("ProfileSummary"));
}
@@ -601,13 +601,6 @@ void Module::setSemanticInterposition(bool SI) {
addModuleFlag(ModFlagBehavior::Error, "SemanticInterposition", SI);
}
-bool Module::noSemanticInterposition() const {
- // Conservatively require an explicit zero value for now.
- Metadata *MF = getModuleFlag("SemanticInterposition");
- auto *Val = cast_or_null<ConstantAsMetadata>(MF);
- return Val && cast<ConstantInt>(Val->getValue())->getZExtValue() == 0;
-}
-
void Module::setOwnedMemoryBuffer(std::unique_ptr<MemoryBuffer> MB) {
OwnedMemoryBuffer = std::move(MB);
}
diff --git a/llvm/lib/IR/ModuleSummaryIndex.cpp b/llvm/lib/IR/ModuleSummaryIndex.cpp
index 91612eafada7..5d21ca759f35 100644
--- a/llvm/lib/IR/ModuleSummaryIndex.cpp
+++ b/llvm/lib/IR/ModuleSummaryIndex.cpp
@@ -163,7 +163,9 @@ bool ModuleSummaryIndex::isGUIDLive(GlobalValue::GUID GUID) const {
return false;
}
-static void propagateAttributesToRefs(GlobalValueSummary *S) {
+static void
+propagateAttributesToRefs(GlobalValueSummary *S,
+ DenseSet<ValueInfo> &MarkedNonReadWriteOnly) {
// If reference is not readonly or writeonly then referenced summary is not
// read/writeonly either. Note that:
// - All references from GlobalVarSummary are conservatively considered as
@@ -174,6 +176,11 @@ static void propagateAttributesToRefs(GlobalValueSummary *S) {
// for them.
for (auto &VI : S->refs()) {
assert(VI.getAccessSpecifier() == 0 || isa<FunctionSummary>(S));
+ if (!VI.getAccessSpecifier()) {
+ if (!MarkedNonReadWriteOnly.insert(VI).second)
+ continue;
+ } else if (MarkedNonReadWriteOnly.contains(VI))
+ continue;
for (auto &Ref : VI.getSummaryList())
// If references to alias is not read/writeonly then aliasee
// is not read/writeonly
@@ -216,11 +223,24 @@ void ModuleSummaryIndex::propagateAttributes(
const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
if (!PropagateAttrs)
return;
+ DenseSet<ValueInfo> MarkedNonReadWriteOnly;
for (auto &P : *this)
for (auto &S : P.second.SummaryList) {
- if (!isGlobalValueLive(S.get()))
+ if (!isGlobalValueLive(S.get())) {
+ // computeDeadSymbols should have marked all copies live. Note that
+ // it is possible that there is a GUID collision between internal
+ // symbols with the same name in different files of the same name but
+ // not enough distinguishing path. Because computeDeadSymbols should
+ // conservatively mark all copies live we can assert here that all are
+ // dead if any copy is dead.
+ assert(llvm::none_of(
+ P.second.SummaryList,
+ [&](const std::unique_ptr<GlobalValueSummary> &Summary) {
+ return isGlobalValueLive(Summary.get());
+ }));
// We don't examine references from dead objects
- continue;
+ break;
+ }
// Global variable can't be marked read/writeonly if it is not eligible
// to import since we need to ensure that all external references get
@@ -240,7 +260,7 @@ void ModuleSummaryIndex::propagateAttributes(
GVS->setReadOnly(false);
GVS->setWriteOnly(false);
}
- propagateAttributesToRefs(S.get());
+ propagateAttributesToRefs(S.get(), MarkedNonReadWriteOnly);
}
setWithAttributePropagation();
if (llvm::AreStatisticsEnabled())
diff --git a/llvm/lib/IR/OptBisect.cpp b/llvm/lib/IR/OptBisect.cpp
index 3104b90f3070..dc85e1316d48 100644
--- a/llvm/lib/IR/OptBisect.cpp
+++ b/llvm/lib/IR/OptBisect.cpp
@@ -54,3 +54,5 @@ bool OptBisect::checkPass(const StringRef PassName,
printPassMessage(PassName, CurBisectNum, TargetDesc, ShouldRun);
return ShouldRun;
}
+
+ManagedStatic<OptBisect> llvm::OptBisector;
diff --git a/llvm/lib/IR/Pass.cpp b/llvm/lib/IR/Pass.cpp
index a815da2bdc51..755ea57c63fd 100644
--- a/llvm/lib/IR/Pass.cpp
+++ b/llvm/lib/IR/Pass.cpp
@@ -62,7 +62,7 @@ bool ModulePass::skipModule(Module &M) const {
}
bool Pass::mustPreserveAnalysisID(char &AID) const {
- return Resolver->getAnalysisIfAvailable(&AID, true) != nullptr;
+ return Resolver->getAnalysisIfAvailable(&AID) != nullptr;
}
// dumpPassStructure - Implement the -debug-pass=Structure option
@@ -259,22 +259,23 @@ void AnalysisUsage::setPreservesCFG() {
AnalysisUsage &AnalysisUsage::addPreserved(StringRef Arg) {
const PassInfo *PI = Pass::lookupPassInfo(Arg);
// If the pass exists, preserve it. Otherwise silently do nothing.
- if (PI) Preserved.push_back(PI->getTypeInfo());
+ if (PI)
+ pushUnique(Preserved, PI->getTypeInfo());
return *this;
}
AnalysisUsage &AnalysisUsage::addRequiredID(const void *ID) {
- Required.push_back(ID);
+ pushUnique(Required, ID);
return *this;
}
AnalysisUsage &AnalysisUsage::addRequiredID(char &ID) {
- Required.push_back(&ID);
+ pushUnique(Required, &ID);
return *this;
}
AnalysisUsage &AnalysisUsage::addRequiredTransitiveID(char &ID) {
- Required.push_back(&ID);
- RequiredTransitive.push_back(&ID);
+ pushUnique(Required, &ID);
+ pushUnique(RequiredTransitive, &ID);
return *this;
}
diff --git a/llvm/lib/IR/PassInstrumentation.cpp b/llvm/lib/IR/PassInstrumentation.cpp
index 49cc6ec04d90..56a36db21e28 100644
--- a/llvm/lib/IR/PassInstrumentation.cpp
+++ b/llvm/lib/IR/PassInstrumentation.cpp
@@ -12,10 +12,29 @@
//===----------------------------------------------------------------------===//
#include "llvm/IR/PassInstrumentation.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/IR/PassManager.h"
namespace llvm {
+void PassInstrumentationCallbacks::addClassToPassName(StringRef ClassName,
+ StringRef PassName) {
+ ClassToPassName[ClassName] = PassName.str();
+}
+
+StringRef
+PassInstrumentationCallbacks::getPassNameForClassName(StringRef ClassName) {
+ return ClassToPassName[ClassName];
+}
+
AnalysisKey PassInstrumentationAnalysis::Key;
+bool isSpecialPass(StringRef PassID, const std::vector<StringRef> &Specials) {
+ size_t Pos = PassID.find('<');
+ StringRef Prefix = PassID;
+ if (Pos != StringRef::npos)
+ Prefix = PassID.substr(0, Pos);
+ return any_of(Specials, [Prefix](StringRef S) { return Prefix.endswith(S); });
+}
+
} // namespace llvm
diff --git a/llvm/lib/IR/PassManager.cpp b/llvm/lib/IR/PassManager.cpp
index 624827ff8cd9..4cf7ab2a602b 100644
--- a/llvm/lib/IR/PassManager.cpp
+++ b/llvm/lib/IR/PassManager.cpp
@@ -91,6 +91,54 @@ bool FunctionAnalysisManagerModuleProxy::Result::invalidate(
}
} // namespace llvm
+PreservedAnalyses ModuleToFunctionPassAdaptor::run(Module &M,
+ ModuleAnalysisManager &AM) {
+ FunctionAnalysisManager &FAM =
+ AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+
+ // Request PassInstrumentation from analysis manager, will use it to run
+ // instrumenting callbacks for the passes later.
+ PassInstrumentation PI = AM.getResult<PassInstrumentationAnalysis>(M);
+
+ PreservedAnalyses PA = PreservedAnalyses::all();
+ for (Function &F : M) {
+ if (F.isDeclaration())
+ continue;
+
+ // Check the PassInstrumentation's BeforePass callbacks before running the
+ // pass, skip its execution completely if asked to (callback returns
+ // false).
+ if (!PI.runBeforePass<Function>(*Pass, F))
+ continue;
+
+ PreservedAnalyses PassPA;
+ {
+ TimeTraceScope TimeScope(Pass->name(), F.getName());
+ PassPA = Pass->run(F, FAM);
+ }
+
+ PI.runAfterPass(*Pass, F, PassPA);
+
+ // We know that the function pass couldn't have invalidated any other
+ // function's analyses (that's the contract of a function pass), so
+ // directly handle the function analysis manager's invalidation here.
+ FAM.invalidate(F, PassPA);
+
+ // Then intersect the preserved set so that invalidation of module
+ // analyses will eventually occur when the module pass completes.
+ PA.intersect(std::move(PassPA));
+ }
+
+ // The FunctionAnalysisManagerModuleProxy is preserved because (we assume)
+ // the function passes we ran didn't add or remove any functions.
+ //
+ // We also preserve all analyses on Functions, because we did all the
+ // invalidation we needed to do above.
+ PA.preserveSet<AllAnalysesOn<Function>>();
+ PA.preserve<FunctionAnalysisManagerModuleProxy>();
+ return PA;
+}
+
AnalysisSetKey CFGAnalyses::SetKey;
AnalysisSetKey PreservedAnalyses::AllAnalysesKey;
diff --git a/llvm/lib/IR/PassRegistry.cpp b/llvm/lib/IR/PassRegistry.cpp
index 0572c4fe5237..94f607afec47 100644
--- a/llvm/lib/IR/PassRegistry.cpp
+++ b/llvm/lib/IR/PassRegistry.cpp
@@ -40,14 +40,12 @@ PassRegistry::~PassRegistry() = default;
const PassInfo *PassRegistry::getPassInfo(const void *TI) const {
sys::SmartScopedReader<true> Guard(Lock);
- MapType::const_iterator I = PassInfoMap.find(TI);
- return I != PassInfoMap.end() ? I->second : nullptr;
+ return PassInfoMap.lookup(TI);
}
const PassInfo *PassRegistry::getPassInfo(StringRef Arg) const {
sys::SmartScopedReader<true> Guard(Lock);
- StringMapType::const_iterator I = PassInfoStringMap.find(Arg);
- return I != PassInfoStringMap.end() ? I->second : nullptr;
+ return PassInfoStringMap.lookup(Arg);
}
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/IR/PassTimingInfo.cpp b/llvm/lib/IR/PassTimingInfo.cpp
index 25275e5733ac..d0c1517f480b 100644
--- a/llvm/lib/IR/PassTimingInfo.cpp
+++ b/llvm/lib/IR/PassTimingInfo.cpp
@@ -16,9 +16,7 @@
//===----------------------------------------------------------------------===//
#include "llvm/IR/PassTimingInfo.h"
-#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/StringRef.h"
#include "llvm/IR/PassInstrumentation.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
@@ -26,9 +24,8 @@
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/Mutex.h"
-#include "llvm/Support/Timer.h"
+#include "llvm/Support/TypeName.h"
#include "llvm/Support/raw_ostream.h"
-#include <memory>
#include <string>
using namespace llvm;
@@ -38,11 +35,17 @@ using namespace llvm;
namespace llvm {
bool TimePassesIsEnabled = false;
+bool TimePassesPerRun = false;
static cl::opt<bool, true> EnableTiming(
"time-passes", cl::location(TimePassesIsEnabled), cl::Hidden,
cl::desc("Time each pass, printing elapsed time for each on exit"));
+static cl::opt<bool, true> EnableTimingPerRun(
+ "time-passes-per-run", cl::location(TimePassesPerRun), cl::Hidden,
+ cl::desc("Time each pass run, printing elapsed time for each run on exit"),
+ cl::callback([](const bool &) { TimePassesIsEnabled = true; }));
+
namespace {
namespace legacy {
@@ -168,6 +171,13 @@ void reportAndResetTimings(raw_ostream *OutStream) {
/// Returns the timer for the specified pass invocation of \p PassID.
/// Each time it creates a new timer.
Timer &TimePassesHandler::getPassTimer(StringRef PassID) {
+ if (!PerRun) {
+ TimerVector &Timers = TimingData[PassID];
+ if (Timers.size() == 0)
+ Timers.emplace_back(new Timer(PassID, PassID, TG));
+ return *Timers.front();
+ }
+
// Take a vector of Timers created for this \p PassID and append
// one more timer to it.
TimerVector &Timers = TimingData[PassID];
@@ -182,8 +192,12 @@ Timer &TimePassesHandler::getPassTimer(StringRef PassID) {
return *T;
}
-TimePassesHandler::TimePassesHandler(bool Enabled)
- : TG("pass", "... Pass execution timing report ..."), Enabled(Enabled) {}
+TimePassesHandler::TimePassesHandler(bool Enabled, bool PerRun)
+ : TG("pass", "... Pass execution timing report ..."), Enabled(Enabled),
+ PerRun(PerRun) {}
+
+TimePassesHandler::TimePassesHandler()
+ : TimePassesHandler(TimePassesIsEnabled, TimePassesPerRun) {}
void TimePassesHandler::setOutStream(raw_ostream &Out) {
OutStream = &Out;
@@ -234,30 +248,20 @@ void TimePassesHandler::stopTimer(StringRef PassID) {
MyTimer->stopTimer();
}
-static bool matchPassManager(StringRef PassID) {
- size_t prefix_pos = PassID.find('<');
- if (prefix_pos == StringRef::npos)
- return false;
- StringRef Prefix = PassID.substr(0, prefix_pos);
- return Prefix.endswith("PassManager") || Prefix.endswith("PassAdaptor") ||
- Prefix.endswith("AnalysisManagerProxy");
-}
-
-bool TimePassesHandler::runBeforePass(StringRef PassID) {
- if (matchPassManager(PassID))
- return true;
+void TimePassesHandler::runBeforePass(StringRef PassID) {
+ if (isSpecialPass(PassID,
+ {"PassManager", "PassAdaptor", "AnalysisManagerProxy"}))
+ return;
startTimer(PassID);
LLVM_DEBUG(dbgs() << "after runBeforePass(" << PassID << ")\n");
LLVM_DEBUG(dump());
-
- // we are not going to skip this pass, thus return true.
- return true;
}
void TimePassesHandler::runAfterPass(StringRef PassID) {
- if (matchPassManager(PassID))
+ if (isSpecialPass(PassID,
+ {"PassManager", "PassAdaptor", "AnalysisManagerProxy"}))
return;
stopTimer(PassID);
@@ -270,12 +274,16 @@ void TimePassesHandler::registerCallbacks(PassInstrumentationCallbacks &PIC) {
if (!Enabled)
return;
- PIC.registerBeforePassCallback(
- [this](StringRef P, Any) { return this->runBeforePass(P); });
+ PIC.registerBeforeNonSkippedPassCallback(
+ [this](StringRef P, Any) { this->runBeforePass(P); });
PIC.registerAfterPassCallback(
- [this](StringRef P, Any) { this->runAfterPass(P); });
+ [this](StringRef P, Any, const PreservedAnalyses &) {
+ this->runAfterPass(P);
+ });
PIC.registerAfterPassInvalidatedCallback(
- [this](StringRef P) { this->runAfterPass(P); });
+ [this](StringRef P, const PreservedAnalyses &) {
+ this->runAfterPass(P);
+ });
PIC.registerBeforeAnalysisCallback(
[this](StringRef P, Any) { this->runBeforePass(P); });
PIC.registerAfterAnalysisCallback(
diff --git a/llvm/lib/IR/PrintPasses.cpp b/llvm/lib/IR/PrintPasses.cpp
new file mode 100644
index 000000000000..83b8c93e766f
--- /dev/null
+++ b/llvm/lib/IR/PrintPasses.cpp
@@ -0,0 +1,88 @@
+//===- PrintPasses.cpp ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/IR/PrintPasses.h"
+#include "llvm/Support/CommandLine.h"
+#include <unordered_set>
+
+using namespace llvm;
+
+// Print IR out before/after specified passes.
+static cl::list<std::string>
+ PrintBefore("print-before",
+ llvm::cl::desc("Print IR before specified passes"),
+ cl::CommaSeparated, cl::Hidden);
+
+static cl::list<std::string>
+ PrintAfter("print-after", llvm::cl::desc("Print IR after specified passes"),
+ cl::CommaSeparated, cl::Hidden);
+
+static cl::opt<bool> PrintBeforeAll("print-before-all",
+ llvm::cl::desc("Print IR before each pass"),
+ cl::init(false), cl::Hidden);
+static cl::opt<bool> PrintAfterAll("print-after-all",
+ llvm::cl::desc("Print IR after each pass"),
+ cl::init(false), cl::Hidden);
+
+static cl::opt<bool>
+ PrintModuleScope("print-module-scope",
+ cl::desc("When printing IR for print-[before|after]{-all} "
+ "always print a module IR"),
+ cl::init(false), cl::Hidden);
+
+static cl::list<std::string>
+ PrintFuncsList("filter-print-funcs", cl::value_desc("function names"),
+ cl::desc("Only print IR for functions whose name "
+ "match this for all print-[before|after][-all] "
+ "options"),
+ cl::CommaSeparated, cl::Hidden);
+
+/// This is a helper to determine whether to print IR before or
+/// after a pass.
+
+bool llvm::shouldPrintBeforeSomePass() {
+ return PrintBeforeAll || !PrintBefore.empty();
+}
+
+bool llvm::shouldPrintAfterSomePass() {
+ return PrintAfterAll || !PrintAfter.empty();
+}
+
+static bool shouldPrintBeforeOrAfterPass(StringRef PassID,
+ ArrayRef<std::string> PassesToPrint) {
+ return llvm::is_contained(PassesToPrint, PassID);
+}
+
+bool llvm::shouldPrintBeforeAll() { return PrintBeforeAll; }
+
+bool llvm::shouldPrintAfterAll() { return PrintAfterAll; }
+
+bool llvm::shouldPrintBeforePass(StringRef PassID) {
+ return PrintBeforeAll || shouldPrintBeforeOrAfterPass(PassID, PrintBefore);
+}
+
+bool llvm::shouldPrintAfterPass(StringRef PassID) {
+ return PrintAfterAll || shouldPrintBeforeOrAfterPass(PassID, PrintAfter);
+}
+
+std::vector<std::string> llvm::printBeforePasses() {
+ return std::vector<std::string>(PrintBefore);
+}
+
+std::vector<std::string> llvm::printAfterPasses() {
+ return std::vector<std::string>(PrintAfter);
+}
+
+bool llvm::forcePrintModuleIR() { return PrintModuleScope; }
+
+bool llvm::isFunctionInPrintList(StringRef FunctionName) {
+ static std::unordered_set<std::string> PrintFuncNames(PrintFuncsList.begin(),
+ PrintFuncsList.end());
+ return PrintFuncNames.empty() ||
+ PrintFuncNames.count(std::string(FunctionName));
+}
diff --git a/llvm/lib/IR/ProfileSummary.cpp b/llvm/lib/IR/ProfileSummary.cpp
index ac6bcd9fe3af..453a278a7f3f 100644
--- a/llvm/lib/IR/ProfileSummary.cpp
+++ b/llvm/lib/IR/ProfileSummary.cpp
@@ -259,7 +259,7 @@ void ProfileSummary::printSummary(raw_ostream &OS) {
void ProfileSummary::printDetailedSummary(raw_ostream &OS) {
OS << "Detailed summary:\n";
- for (auto Entry : DetailedSummary) {
+ for (const auto &Entry : DetailedSummary) {
OS << Entry.NumCounts << " blocks with count >= " << Entry.MinCount
<< " account for "
<< format("%0.6g", (float)Entry.Cutoff / Scale * 100)
diff --git a/llvm/lib/IR/PseudoProbe.cpp b/llvm/lib/IR/PseudoProbe.cpp
new file mode 100644
index 000000000000..804214f06e7a
--- /dev/null
+++ b/llvm/lib/IR/PseudoProbe.cpp
@@ -0,0 +1,58 @@
+//===- PseudoProbe.cpp - Pseudo Probe Helpers -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the helpers to manipulate pseudo probe IR intrinsic
+// calls.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/IR/PseudoProbe.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instruction.h"
+
+using namespace llvm;
+
+namespace llvm {
+
+Optional<PseudoProbe> extractProbeFromDiscriminator(const Instruction &Inst) {
+ assert(isa<CallBase>(&Inst) && !isa<IntrinsicInst>(&Inst) &&
+ "Only call instructions should have pseudo probe encodes as their "
+ "Dwarf discriminators");
+ if (const DebugLoc &DLoc = Inst.getDebugLoc()) {
+ const DILocation *DIL = DLoc;
+ auto Discriminator = DIL->getDiscriminator();
+ if (DILocation::isPseudoProbeDiscriminator(Discriminator)) {
+ PseudoProbe Probe;
+ Probe.Id =
+ PseudoProbeDwarfDiscriminator::extractProbeIndex(Discriminator);
+ Probe.Type =
+ PseudoProbeDwarfDiscriminator::extractProbeType(Discriminator);
+ Probe.Attr =
+ PseudoProbeDwarfDiscriminator::extractProbeAttributes(Discriminator);
+ return Probe;
+ }
+ }
+ return None;
+}
+
+Optional<PseudoProbe> extractProbe(const Instruction &Inst) {
+ if (const auto *II = dyn_cast<PseudoProbeInst>(&Inst)) {
+ PseudoProbe Probe;
+ Probe.Id = II->getIndex()->getZExtValue();
+ Probe.Type = (uint32_t)PseudoProbeType::Block;
+ Probe.Attr = II->getAttributes()->getZExtValue();
+ return Probe;
+ }
+
+ if (isa<CallBase>(&Inst) && !isa<IntrinsicInst>(&Inst))
+ return extractProbeFromDiscriminator(Inst);
+
+ return None;
+}
+} // namespace llvm
diff --git a/llvm/lib/IR/ReplaceConstant.cpp b/llvm/lib/IR/ReplaceConstant.cpp
new file mode 100644
index 000000000000..7efa525d427e
--- /dev/null
+++ b/llvm/lib/IR/ReplaceConstant.cpp
@@ -0,0 +1,70 @@
+//===- ReplaceConstant.cpp - Replace LLVM constant expression--------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a utility function for replacing LLVM constant
+// expressions by instructions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/IR/ReplaceConstant.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/NoFolder.h"
+
+namespace llvm {
+// Replace a constant expression by instructions with equivalent operations at
+// a specified location.
+Instruction *createReplacementInstr(ConstantExpr *CE, Instruction *Instr) {
+ IRBuilder<NoFolder> Builder(Instr);
+ unsigned OpCode = CE->getOpcode();
+ switch (OpCode) {
+ case Instruction::GetElementPtr: {
+ SmallVector<Value *, 4> CEOpVec(CE->operands());
+ ArrayRef<Value *> CEOps(CEOpVec);
+ return dyn_cast<Instruction>(
+ Builder.CreateInBoundsGEP(cast<GEPOperator>(CE)->getSourceElementType(),
+ CEOps[0], CEOps.slice(1)));
+ }
+ case Instruction::Add:
+ case Instruction::Sub:
+ case Instruction::Mul:
+ case Instruction::UDiv:
+ case Instruction::SDiv:
+ case Instruction::FDiv:
+ case Instruction::URem:
+ case Instruction::SRem:
+ case Instruction::FRem:
+ case Instruction::Shl:
+ case Instruction::LShr:
+ case Instruction::AShr:
+ case Instruction::And:
+ case Instruction::Or:
+ case Instruction::Xor:
+ return dyn_cast<Instruction>(
+ Builder.CreateBinOp((Instruction::BinaryOps)OpCode, CE->getOperand(0),
+ CE->getOperand(1), CE->getName()));
+ case Instruction::Trunc:
+ case Instruction::ZExt:
+ case Instruction::SExt:
+ case Instruction::FPToUI:
+ case Instruction::FPToSI:
+ case Instruction::UIToFP:
+ case Instruction::SIToFP:
+ case Instruction::FPTrunc:
+ case Instruction::FPExt:
+ case Instruction::PtrToInt:
+ case Instruction::IntToPtr:
+ case Instruction::BitCast:
+ return dyn_cast<Instruction>(
+ Builder.CreateCast((Instruction::CastOps)OpCode, CE->getOperand(0),
+ CE->getType(), CE->getName()));
+ default:
+ llvm_unreachable("Unhandled constant expression!\n");
+ }
+}
+} // namespace llvm
diff --git a/llvm/lib/IR/SafepointIRVerifier.cpp b/llvm/lib/IR/SafepointIRVerifier.cpp
index 6bf7caa50a12..8ed31b6668e0 100644
--- a/llvm/lib/IR/SafepointIRVerifier.cpp
+++ b/llvm/lib/IR/SafepointIRVerifier.cpp
@@ -561,8 +561,7 @@ GCPtrTracker::GCPtrTracker(const Function &F, const DominatorTree &DT,
}
BasicBlockState *GCPtrTracker::getBasicBlockState(const BasicBlock *BB) {
- auto it = BlockMap.find(BB);
- return it != BlockMap.end() ? it->second : nullptr;
+ return BlockMap.lookup(BB);
}
const BasicBlockState *GCPtrTracker::getBasicBlockState(
diff --git a/llvm/lib/IR/StructuralHash.cpp b/llvm/lib/IR/StructuralHash.cpp
new file mode 100644
index 000000000000..5a6e07451326
--- /dev/null
+++ b/llvm/lib/IR/StructuralHash.cpp
@@ -0,0 +1,84 @@
+//===-- StructuralHash.cpp - IR Hash for expensive checks -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+
+#ifdef EXPENSIVE_CHECKS
+
+#include "llvm/IR/StructuralHash.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Module.h"
+
+using namespace llvm;
+
+namespace {
+namespace details {
+
+// Basic hashing mechanism to detect structural change to the IR, used to verify
+// pass return status consistency with actual change. Loosely copied from
+// llvm/lib/Transforms/Utils/FunctionComparator.cpp
+
+class StructuralHash {
+ uint64_t Hash = 0x6acaa36bef8325c5ULL;
+
+ void update(uint64_t V) { Hash = hashing::detail::hash_16_bytes(Hash, V); }
+
+public:
+ StructuralHash() = default;
+
+ void update(const Function &F) {
+ if (F.empty())
+ return;
+
+ update(F.isVarArg());
+ update(F.arg_size());
+
+ SmallVector<const BasicBlock *, 8> BBs;
+ SmallPtrSet<const BasicBlock *, 16> VisitedBBs;
+
+ BBs.push_back(&F.getEntryBlock());
+ VisitedBBs.insert(BBs[0]);
+ while (!BBs.empty()) {
+ const BasicBlock *BB = BBs.pop_back_val();
+ update(45798); // Block header
+ for (auto &Inst : *BB)
+ update(Inst.getOpcode());
+
+ const Instruction *Term = BB->getTerminator();
+ for (unsigned i = 0, e = Term->getNumSuccessors(); i != e; ++i) {
+ if (!VisitedBBs.insert(Term->getSuccessor(i)).second)
+ continue;
+ BBs.push_back(Term->getSuccessor(i));
+ }
+ }
+ }
+
+ void update(const Module &M) {
+ for (const Function &F : M)
+ update(F);
+ }
+
+ uint64_t getHash() const { return Hash; }
+};
+
+} // namespace details
+
+} // namespace
+
+uint64_t llvm::StructuralHash(const Function &F) {
+ details::StructuralHash H;
+ H.update(F);
+ return H.getHash();
+}
+
+uint64_t llvm::StructuralHash(const Module &M) {
+ details::StructuralHash H;
+ H.update(M);
+ return H.getHash();
+}
+
+#endif
diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp
index d869a6e07cca..bade7dc325f4 100644
--- a/llvm/lib/IR/Type.cpp
+++ b/llvm/lib/IR/Type.cpp
@@ -49,6 +49,7 @@ Type *Type::getPrimitiveType(LLVMContext &C, TypeID IDNumber) {
case LabelTyID : return getLabelTy(C);
case MetadataTyID : return getMetadataTy(C);
case X86_MMXTyID : return getX86_MMXTy(C);
+ case X86_AMXTyID : return getX86_AMXTy(C);
case TokenTyID : return getTokenTy(C);
default:
return nullptr;
@@ -81,6 +82,14 @@ bool Type::canLosslesslyBitCastTo(Type *Ty) const {
Ty->getPrimitiveSizeInBits().getFixedSize() == 64)
return true;
+ // 8192-bit fixed width vector types can be losslessly converted to x86amx.
+ if (((isa<FixedVectorType>(this)) && Ty->isX86_AMXTy()) &&
+ getPrimitiveSizeInBits().getFixedSize() == 8192)
+ return true;
+ if ((isX86_AMXTy() && isa<FixedVectorType>(Ty)) &&
+ Ty->getPrimitiveSizeInBits().getFixedSize() == 8192)
+ return true;
+
// At this point we have only various mismatches of the first class types
// remaining and ptr->ptr. Just select the lossless conversions. Everything
// else is not lossless. Conservatively assume we can't losslessly convert
@@ -120,6 +129,7 @@ TypeSize Type::getPrimitiveSizeInBits() const {
case Type::FP128TyID: return TypeSize::Fixed(128);
case Type::PPC_FP128TyID: return TypeSize::Fixed(128);
case Type::X86_MMXTyID: return TypeSize::Fixed(64);
+ case Type::X86_AMXTyID: return TypeSize::Fixed(8192);
case Type::IntegerTyID:
return TypeSize::Fixed(cast<IntegerType>(this)->getBitWidth());
case Type::FixedVectorTyID:
@@ -128,7 +138,7 @@ TypeSize Type::getPrimitiveSizeInBits() const {
ElementCount EC = VTy->getElementCount();
TypeSize ETS = VTy->getElementType()->getPrimitiveSizeInBits();
assert(!ETS.isScalable() && "Vector type should have fixed-width elements");
- return {ETS.getFixedSize() * EC.Min, EC.Scalable};
+ return {ETS.getFixedSize() * EC.getKnownMinValue(), EC.isScalable()};
}
default: return TypeSize::Fixed(0);
}
@@ -179,6 +189,7 @@ Type *Type::getX86_FP80Ty(LLVMContext &C) { return &C.pImpl->X86_FP80Ty; }
Type *Type::getFP128Ty(LLVMContext &C) { return &C.pImpl->FP128Ty; }
Type *Type::getPPC_FP128Ty(LLVMContext &C) { return &C.pImpl->PPC_FP128Ty; }
Type *Type::getX86_MMXTy(LLVMContext &C) { return &C.pImpl->X86_MMXTy; }
+Type *Type::getX86_AMXTy(LLVMContext &C) { return &C.pImpl->X86_AMXTy; }
IntegerType *Type::getInt1Ty(LLVMContext &C) { return &C.pImpl->Int1Ty; }
IntegerType *Type::getInt8Ty(LLVMContext &C) { return &C.pImpl->Int8Ty; }
@@ -223,6 +234,10 @@ PointerType *Type::getX86_MMXPtrTy(LLVMContext &C, unsigned AS) {
return getX86_MMXTy(C)->getPointerTo(AS);
}
+PointerType *Type::getX86_AMXPtrTy(LLVMContext &C, unsigned AS) {
+ return getX86_AMXTy(C)->getPointerTo(AS);
+}
+
PointerType *Type::getIntNPtrTy(LLVMContext &C, unsigned N, unsigned AS) {
return getIntNTy(C, N)->getPointerTo(AS);
}
@@ -275,11 +290,6 @@ IntegerType *IntegerType::get(LLVMContext &C, unsigned NumBits) {
return Entry;
}
-bool IntegerType::isPowerOf2ByteWidth() const {
- unsigned BitWidth = getBitWidth();
- return (BitWidth > 7) && isPowerOf2_32(BitWidth);
-}
-
APInt IntegerType::getMask() const {
return APInt::getAllOnesValue(getBitWidth());
}
@@ -380,6 +390,18 @@ StructType *StructType::get(LLVMContext &Context, ArrayRef<Type*> ETypes,
return ST;
}
+bool StructType::containsScalableVectorType() const {
+ for (Type *Ty : elements()) {
+ if (isa<ScalableVectorType>(Ty))
+ return true;
+ if (auto *STy = dyn_cast<StructType>(Ty))
+ if (STy->containsScalableVectorType())
+ return true;
+ }
+
+ return false;
+}
+
void StructType::setBody(ArrayRef<Type*> Elements, bool isPacked) {
assert(isOpaque() && "Struct body already set!");
@@ -499,9 +521,14 @@ bool StructType::isSized(SmallPtrSetImpl<Type*> *Visited) const {
// Okay, our struct is sized if all of the elements are, but if one of the
// elements is opaque, the struct isn't sized *yet*, but may become sized in
// the future, so just bail out without caching.
- for (element_iterator I = element_begin(), E = element_end(); I != E; ++I)
- if (!(*I)->isSized(Visited))
+ for (Type *Ty : elements()) {
+ // If the struct contains a scalable vector type, don't consider it sized.
+ // This prevents it from being used in loads/stores/allocas/GEPs.
+ if (isa<ScalableVectorType>(Ty))
+ return false;
+ if (!Ty->isSized(Visited))
return false;
+ }
// Here we cheat a bit and cast away const-ness. The goal is to memoize when
// we find a sized type, as types can only move from opaque to sized, not the
@@ -521,7 +548,7 @@ StringRef StructType::getName() const {
bool StructType::isValidElementType(Type *ElemTy) {
return !ElemTy->isVoidTy() && !ElemTy->isLabelTy() &&
!ElemTy->isMetadataTy() && !ElemTy->isFunctionTy() &&
- !ElemTy->isTokenTy() && !isa<ScalableVectorType>(ElemTy);
+ !ElemTy->isTokenTy();
}
bool StructType::isLayoutIdentical(StructType *Other) const {
@@ -533,10 +560,6 @@ bool StructType::isLayoutIdentical(StructType *Other) const {
return elements() == Other->elements();
}
-StructType *Module::getTypeByName(StringRef Name) const {
- return getContext().pImpl->NamedStructTypes.lookup(Name);
-}
-
Type *StructType::getTypeAtIndex(const Value *V) const {
unsigned Idx = (unsigned)cast<Constant>(V)->getUniqueInteger().getZExtValue();
assert(indexValid(Idx) && "Invalid structure index!");
@@ -557,6 +580,10 @@ bool StructType::indexValid(const Value *V) const {
return CU && CU->getZExtValue() < getNumElements();
}
+StructType *StructType::getTypeByName(LLVMContext &C, StringRef Name) {
+ return C.pImpl->NamedStructTypes.lookup(Name);
+}
+
//===----------------------------------------------------------------------===//
// ArrayType Implementation
//===----------------------------------------------------------------------===//
@@ -598,10 +625,10 @@ VectorType::VectorType(Type *ElType, unsigned EQ, Type::TypeID TID)
}
VectorType *VectorType::get(Type *ElementType, ElementCount EC) {
- if (EC.Scalable)
- return ScalableVectorType::get(ElementType, EC.Min);
+ if (EC.isScalable())
+ return ScalableVectorType::get(ElementType, EC.getKnownMinValue());
else
- return FixedVectorType::get(ElementType, EC.Min);
+ return FixedVectorType::get(ElementType, EC.getKnownMinValue());
}
bool VectorType::isValidElementType(Type *ElemTy) {
@@ -619,7 +646,7 @@ FixedVectorType *FixedVectorType::get(Type *ElementType, unsigned NumElts) {
"be an integer, floating point, or "
"pointer type.");
- ElementCount EC(NumElts, false);
+ auto EC = ElementCount::getFixed(NumElts);
LLVMContextImpl *pImpl = ElementType->getContext().pImpl;
VectorType *&Entry = ElementType->getContext()
@@ -641,7 +668,7 @@ ScalableVectorType *ScalableVectorType::get(Type *ElementType,
"be an integer, floating point, or "
"pointer type.");
- ElementCount EC(MinNumElts, true);
+ auto EC = ElementCount::getScalable(MinNumElts);
LLVMContextImpl *pImpl = ElementType->getContext().pImpl;
VectorType *&Entry = ElementType->getContext()
diff --git a/llvm/lib/IR/Use.cpp b/llvm/lib/IR/Use.cpp
index dc0716b85372..99049c0232aa 100644
--- a/llvm/lib/IR/Use.cpp
+++ b/llvm/lib/IR/Use.cpp
@@ -17,24 +17,17 @@ void Use::swap(Use &RHS) {
if (Val == RHS.Val)
return;
- if (Val)
- removeFromList();
-
- Value *OldVal = Val;
- if (RHS.Val) {
- RHS.removeFromList();
- Val = RHS.Val;
- Val->addUse(*this);
- } else {
- Val = nullptr;
- }
-
- if (OldVal) {
- RHS.Val = OldVal;
- RHS.Val->addUse(RHS);
- } else {
- RHS.Val = nullptr;
- }
+ std::swap(Val, RHS.Val);
+ std::swap(Next, RHS.Next);
+ std::swap(Prev, RHS.Prev);
+
+ *Prev = this;
+ if (Next)
+ Next->Prev = &Next;
+
+ *RHS.Prev = &RHS;
+ if (RHS.Next)
+ RHS.Next->Prev = &RHS.Next;
}
unsigned Use::getOperandNo() const {
diff --git a/llvm/lib/IR/User.cpp b/llvm/lib/IR/User.cpp
index 7da592f40127..9105c6fbd230 100644
--- a/llvm/lib/IR/User.cpp
+++ b/llvm/lib/IR/User.cpp
@@ -29,7 +29,7 @@ void User::replaceUsesOfWith(Value *From, Value *To) {
// The side effects of this setOperand call include linking to
// "To", adding "this" to the uses list of To, and
// most importantly, removing "this" from the use list of "From".
- setOperand(i, To); // Fix it now...
+ setOperand(i, To);
}
}
diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp
index efb8d53e8964..572f37a32410 100644
--- a/llvm/lib/IR/Value.cpp
+++ b/llvm/lib/IR/Value.cpp
@@ -51,9 +51,9 @@ static inline Type *checkType(Type *Ty) {
}
Value::Value(Type *ty, unsigned scid)
- : VTy(checkType(ty)), UseList(nullptr), SubclassID(scid),
- HasValueHandle(0), SubclassOptionalData(0), SubclassData(0),
- NumUserOperands(0), IsUsedByMD(false), HasName(false) {
+ : VTy(checkType(ty)), UseList(nullptr), SubclassID(scid), HasValueHandle(0),
+ SubclassOptionalData(0), SubclassData(0), NumUserOperands(0),
+ IsUsedByMD(false), HasName(false), HasMetadata(false) {
static_assert(ConstantFirstVal == 0, "!(SubclassID < ConstantFirstVal)");
// FIXME: Why isn't this in the subclass gunk??
// Note, we cannot call isa<CallInst> before the CallInst has been
@@ -77,6 +77,10 @@ Value::~Value() {
if (isUsedByMetadata())
ValueAsMetadata::handleDeletion(this);
+ // Remove associated metadata from context.
+ if (HasMetadata)
+ clearMetadata();
+
#ifndef NDEBUG // Only in -g mode...
// Check to make sure that there are no uses of this value that are still
// around when the value is destroyed. If there are, then we have a dangling
@@ -147,6 +151,14 @@ bool Value::hasNUsesOrMore(unsigned N) const {
return hasNItemsOrMore(use_begin(), use_end(), N);
}
+bool Value::hasOneUser() const {
+ if (use_empty())
+ return false;
+ if (hasOneUse())
+ return true;
+ return std::equal(++user_begin(), user_end(), user_begin());
+}
+
static bool isUnDroppableUser(const User *U) { return !U->isDroppable(); }
Use *Value::getSingleUndroppableUse() {
@@ -175,21 +187,34 @@ void Value::dropDroppableUses(
for (Use &U : uses())
if (U.getUser()->isDroppable() && ShouldDrop(&U))
ToBeEdited.push_back(&U);
- for (Use *U : ToBeEdited) {
- U->removeFromList();
- if (auto *Assume = dyn_cast<IntrinsicInst>(U->getUser())) {
- assert(Assume->getIntrinsicID() == Intrinsic::assume);
- unsigned OpNo = U->getOperandNo();
- if (OpNo == 0)
- Assume->setOperand(0, ConstantInt::getTrue(Assume->getContext()));
- else {
- Assume->setOperand(OpNo, UndefValue::get(U->get()->getType()));
- CallInst::BundleOpInfo &BOI = Assume->getBundleOpInfoForOperand(OpNo);
- BOI.Tag = getContext().pImpl->getOrInsertBundleTag("ignore");
- }
- } else
- llvm_unreachable("unkown droppable use");
+ for (Use *U : ToBeEdited)
+ dropDroppableUse(*U);
+}
+
+void Value::dropDroppableUsesIn(User &Usr) {
+ assert(Usr.isDroppable() && "Expected a droppable user!");
+ for (Use &UsrOp : Usr.operands()) {
+ if (UsrOp.get() == this)
+ dropDroppableUse(UsrOp);
+ }
+}
+
+void Value::dropDroppableUse(Use &U) {
+ U.removeFromList();
+ if (auto *Assume = dyn_cast<IntrinsicInst>(U.getUser())) {
+ assert(Assume->getIntrinsicID() == Intrinsic::assume);
+ unsigned OpNo = U.getOperandNo();
+ if (OpNo == 0)
+ U.set(ConstantInt::getTrue(Assume->getContext()));
+ else {
+ U.set(UndefValue::get(U.get()->getType()));
+ CallInst::BundleOpInfo &BOI = Assume->getBundleOpInfoForOperand(OpNo);
+ BOI.Tag = Assume->getContext().pImpl->getOrInsertBundleTag("ignore");
+ }
+ return;
}
+
+ llvm_unreachable("unkown droppable use");
}
bool Value::isUsedInBasicBlock(const BasicBlock *BB) const {
@@ -405,6 +430,18 @@ void Value::takeName(Value *V) {
ST->reinsertValue(this);
}
+#ifndef NDEBUG
+std::string Value::getNameOrAsOperand() const {
+ if (!getName().empty())
+ return std::string(getName());
+
+ std::string BBName;
+ raw_string_ostream OS(BBName);
+ printAsOperand(OS, false);
+ return OS.str();
+}
+#endif
+
void Value::assertModuleIsMaterializedImpl() const {
#ifndef NDEBUG
const GlobalValue *GV = dyn_cast<GlobalValue>(this);
@@ -691,11 +728,16 @@ uint64_t Value::getPointerDereferenceableBytes(const DataLayout &DL,
CanBeNull = false;
if (const Argument *A = dyn_cast<Argument>(this)) {
DerefBytes = A->getDereferenceableBytes();
- if (DerefBytes == 0 && (A->hasByValAttr() || A->hasStructRetAttr())) {
- Type *PT = cast<PointerType>(A->getType())->getElementType();
- if (PT->isSized())
- DerefBytes = DL.getTypeStoreSize(PT).getKnownMinSize();
+ if (DerefBytes == 0) {
+ // Handle byval/byref/inalloca/preallocated arguments
+ if (Type *ArgMemTy = A->getPointeeInMemoryValueType()) {
+ if (ArgMemTy->isSized()) {
+ // FIXME: Why isn't this the type alloc size?
+ DerefBytes = DL.getTypeStoreSize(ArgMemTy).getKnownMinSize();
+ }
+ }
}
+
if (DerefBytes == 0) {
DerefBytes = A->getDereferenceableOrNullBytes();
CanBeNull = true;
@@ -783,7 +825,7 @@ Align Value::getPointerAlignment(const DataLayout &DL) const {
const MaybeAlign Alignment = A->getParamAlign();
if (!Alignment && A->hasStructRetAttr()) {
// An sret parameter has at least the ABI alignment of the return type.
- Type *EltTy = cast<PointerType>(A->getType())->getElementType();
+ Type *EltTy = A->getParamStructRetType();
if (EltTy->isSized())
return DL.getABITypeAlign(EltTy);
}
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 6df1072925f9..100e881c8fa8 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -115,6 +115,11 @@
using namespace llvm;
+static cl::opt<bool> VerifyNoAliasScopeDomination(
+ "verify-noalias-scope-decl-dom", cl::Hidden, cl::init(false),
+ cl::desc("Ensure that llvm.experimental.noalias.scope.decl for identical "
+ "scopes are not dominating"));
+
namespace llvm {
struct VerifierSupport {
@@ -282,6 +287,9 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport {
/// Whether the current function has a DISubprogram attached to it.
bool HasDebugInfo = false;
+ /// The current source language.
+ dwarf::SourceLanguage CurrentSourceLang = dwarf::DW_LANG_lo_user;
+
/// Whether source was present on the first DIFile encountered in each CU.
DenseMap<const DICompileUnit *, bool> HasSourceDebugInfo;
@@ -310,6 +318,8 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport {
TBAAVerifier TBAAVerifyHelper;
+ SmallVector<IntrinsicInst *, 4> NoAliasScopeDecls;
+
void checkAtomicMemAccessSize(Type *Ty, const Instruction *I);
public:
@@ -357,6 +367,8 @@ public:
LandingPadResultTy = nullptr;
SawFrameEscape = false;
SiblingFuncletInfo.clear();
+ verifyNoAliasScopeDecl();
+ NoAliasScopeDecls.clear();
return !Broken;
}
@@ -424,6 +436,7 @@ private:
void visitRangeMetadata(Instruction &I, MDNode *Range, Type *Ty);
void visitDereferenceableMetadata(Instruction &I, MDNode *MD);
void visitProfMetadata(Instruction &I, MDNode *MD);
+ void visitAnnotationMetadata(MDNode *Annotation);
template <class Ty> bool isValidMetadataArray(const MDTuple &N);
#define HANDLE_SPECIALIZED_MDNODE_LEAF(CLASS) void visit##CLASS(const CLASS &N);
@@ -501,8 +514,6 @@ private:
void verifySwiftErrorCall(CallBase &Call, const Value *SwiftErrorVal);
void verifySwiftErrorValue(const Value *SwiftErrorVal);
void verifyMustTailCall(CallInst &CI);
- bool performTypeCheck(Intrinsic::ID ID, Function *F, Type *Ty, int VT,
- unsigned ArgNo, std::string &Suffix);
bool verifyAttributeCount(AttributeList Attrs, unsigned Params);
void verifyAttributeTypes(AttributeSet Attrs, bool IsFunction,
const Value *V);
@@ -534,6 +545,9 @@ private:
/// Verify all-or-nothing property of DIFile source attribute within a CU.
void verifySourceDebugInfo(const DICompileUnit &U, const DIFile &F);
+
+ /// Verify the llvm.experimental.noalias.scope.decl declarations
+ void verifyNoAliasScopeDecl();
};
} // end anonymous namespace
@@ -589,7 +603,8 @@ void Verifier::visitGlobalValue(const GlobalValue &GV) {
Assert(!GV.isDSOLocal(),
"GlobalValue with DLLImport Storage is dso_local!", &GV);
- Assert((GV.isDeclaration() && GV.hasExternalLinkage()) ||
+ Assert((GV.isDeclaration() &&
+ (GV.hasExternalLinkage() || GV.hasExternalWeakLinkage())) ||
GV.hasAvailableExternallyLinkage(),
"Global is marked as dllimport, but not external", &GV);
}
@@ -699,12 +714,16 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) {
}
// Scalable vectors cannot be global variables, since we don't know
- // the runtime size. If the global is a struct or an array containing
- // scalable vectors, that will be caught by the isValidElementType methods
- // in StructType or ArrayType instead.
+ // the runtime size. If the global is an array containing scalable vectors,
+ // that will be caught by the isValidElementType methods in StructType or
+ // ArrayType instead.
Assert(!isa<ScalableVectorType>(GV.getValueType()),
"Globals cannot contain scalable vectors", &GV);
+ if (auto *STy = dyn_cast<StructType>(GV.getValueType()))
+ Assert(!STy->containsScalableVectorType(),
+ "Globals cannot contain scalable vectors", &GV);
+
if (!GV.hasInitializer()) {
visitGlobalValue(GV);
return;
@@ -894,7 +913,9 @@ void Verifier::visitDIScope(const DIScope &N) {
void Verifier::visitDISubrange(const DISubrange &N) {
AssertDI(N.getTag() == dwarf::DW_TAG_subrange_type, "invalid tag", &N);
- AssertDI(N.getRawCountNode() || N.getRawUpperBound(),
+ bool HasAssumedSizedArraySupport = dwarf::isFortran(CurrentSourceLang);
+ AssertDI(HasAssumedSizedArraySupport || N.getRawCountNode() ||
+ N.getRawUpperBound(),
"Subrange must contain count or upperBound", &N);
AssertDI(!N.getRawCountNode() || !N.getRawUpperBound(),
"Subrange can have any one of count or upperBound", &N);
@@ -920,14 +941,43 @@ void Verifier::visitDISubrange(const DISubrange &N) {
"Stride must be signed constant or DIVariable or DIExpression", &N);
}
+void Verifier::visitDIGenericSubrange(const DIGenericSubrange &N) {
+ AssertDI(N.getTag() == dwarf::DW_TAG_generic_subrange, "invalid tag", &N);
+ AssertDI(N.getRawCountNode() || N.getRawUpperBound(),
+ "GenericSubrange must contain count or upperBound", &N);
+ AssertDI(!N.getRawCountNode() || !N.getRawUpperBound(),
+ "GenericSubrange can have any one of count or upperBound", &N);
+ auto *CBound = N.getRawCountNode();
+ AssertDI(!CBound || isa<DIVariable>(CBound) || isa<DIExpression>(CBound),
+ "Count must be signed constant or DIVariable or DIExpression", &N);
+ auto *LBound = N.getRawLowerBound();
+ AssertDI(LBound, "GenericSubrange must contain lowerBound", &N);
+ AssertDI(isa<DIVariable>(LBound) || isa<DIExpression>(LBound),
+ "LowerBound must be signed constant or DIVariable or DIExpression",
+ &N);
+ auto *UBound = N.getRawUpperBound();
+ AssertDI(!UBound || isa<DIVariable>(UBound) || isa<DIExpression>(UBound),
+ "UpperBound must be signed constant or DIVariable or DIExpression",
+ &N);
+ auto *Stride = N.getRawStride();
+ AssertDI(Stride, "GenericSubrange must contain stride", &N);
+ AssertDI(isa<DIVariable>(Stride) || isa<DIExpression>(Stride),
+ "Stride must be signed constant or DIVariable or DIExpression", &N);
+}
+
void Verifier::visitDIEnumerator(const DIEnumerator &N) {
AssertDI(N.getTag() == dwarf::DW_TAG_enumerator, "invalid tag", &N);
}
void Verifier::visitDIBasicType(const DIBasicType &N) {
AssertDI(N.getTag() == dwarf::DW_TAG_base_type ||
- N.getTag() == dwarf::DW_TAG_unspecified_type,
+ N.getTag() == dwarf::DW_TAG_unspecified_type ||
+ N.getTag() == dwarf::DW_TAG_string_type,
"invalid tag", &N);
+}
+
+void Verifier::visitDIStringType(const DIStringType &N) {
+ AssertDI(N.getTag() == dwarf::DW_TAG_string_type, "invalid tag", &N);
AssertDI(!(N.isBigEndian() && N.isLittleEndian()) ,
"has conflicting flags", &N);
}
@@ -1035,6 +1085,21 @@ void Verifier::visitDICompositeType(const DICompositeType &N) {
AssertDI(N.getTag() == dwarf::DW_TAG_array_type,
"dataLocation can only appear in array type");
}
+
+ if (N.getRawAssociated()) {
+ AssertDI(N.getTag() == dwarf::DW_TAG_array_type,
+ "associated can only appear in array type");
+ }
+
+ if (N.getRawAllocated()) {
+ AssertDI(N.getTag() == dwarf::DW_TAG_array_type,
+ "allocated can only appear in array type");
+ }
+
+ if (N.getRawRank()) {
+ AssertDI(N.getTag() == dwarf::DW_TAG_array_type,
+ "rank can only appear in array type");
+ }
}
void Verifier::visitDISubroutineType(const DISubroutineType &N) {
@@ -1084,6 +1149,8 @@ void Verifier::visitDICompileUnit(const DICompileUnit &N) {
AssertDI(!N.getFile()->getFilename().empty(), "invalid filename", &N,
N.getFile());
+ CurrentSourceLang = (dwarf::SourceLanguage)N.getSourceLanguage();
+
verifySourceDebugInfo(N, *N.getFile());
AssertDI((N.getEmissionKind() <= DICompileUnit::LastEmissionKind),
@@ -1525,8 +1592,8 @@ void Verifier::visitModuleFlagCGProfileEntry(const MDOperand &MDO) {
if (!FuncMDO)
return;
auto F = dyn_cast<ValueAsMetadata>(FuncMDO);
- Assert(F && isa<Function>(F->getValue()), "expected a Function or null",
- FuncMDO);
+ Assert(F && isa<Function>(F->getValue()->stripPointerCasts()),
+ "expected a Function or null", FuncMDO);
};
auto Node = dyn_cast_or_null<MDNode>(MDO);
Assert(Node && Node->getNumOperands() == 3, "expected a MDNode triple", MDO);
@@ -1544,6 +1611,7 @@ static bool isFuncOnlyAttr(Attribute::AttrKind Kind) {
case Attribute::NoReturn:
case Attribute::NoSync:
case Attribute::WillReturn:
+ case Attribute::NoCallback:
case Attribute::NoCfCheck:
case Attribute::NoUnwind:
case Attribute::NoInline:
@@ -1572,6 +1640,7 @@ static bool isFuncOnlyAttr(Attribute::AttrKind Kind) {
case Attribute::Builtin:
case Attribute::NoBuiltin:
case Attribute::Cold:
+ case Attribute::Hot:
case Attribute::OptForFuzzing:
case Attribute::OptimizeNone:
case Attribute::JumpTable:
@@ -1585,6 +1654,8 @@ static bool isFuncOnlyAttr(Attribute::AttrKind Kind) {
case Attribute::Speculatable:
case Attribute::StrictFP:
case Attribute::NullPointerIsValid:
+ case Attribute::MustProgress:
+ case Attribute::NoProfile:
return true;
default:
break;
@@ -1652,9 +1723,10 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty,
AttrCount += Attrs.hasAttribute(Attribute::StructRet) ||
Attrs.hasAttribute(Attribute::InReg);
AttrCount += Attrs.hasAttribute(Attribute::Nest);
+ AttrCount += Attrs.hasAttribute(Attribute::ByRef);
Assert(AttrCount <= 1,
"Attributes 'byval', 'inalloca', 'preallocated', 'inreg', 'nest', "
- "and 'sret' are incompatible!",
+ "'byref', and 'sret' are incompatible!",
V);
Assert(!(Attrs.hasAttribute(Attribute::InAlloca) &&
@@ -1699,17 +1771,6 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty,
"'noinline and alwaysinline' are incompatible!",
V);
- if (Attrs.hasAttribute(Attribute::ByVal) && Attrs.getByValType()) {
- Assert(Attrs.getByValType() == cast<PointerType>(Ty)->getElementType(),
- "Attribute 'byval' type does not match parameter!", V);
- }
-
- if (Attrs.hasAttribute(Attribute::Preallocated)) {
- Assert(Attrs.getPreallocatedType() ==
- cast<PointerType>(Ty)->getElementType(),
- "Attribute 'preallocated' type does not match parameter!", V);
- }
-
AttrBuilder IncompatibleAttrs = AttributeFuncs::typeIncompatible(Ty);
Assert(!AttrBuilder(Attrs).overlaps(IncompatibleAttrs),
"Wrong types for attribute: " +
@@ -1720,9 +1781,10 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty,
SmallPtrSet<Type*, 4> Visited;
if (!PTy->getElementType()->isSized(&Visited)) {
Assert(!Attrs.hasAttribute(Attribute::ByVal) &&
- !Attrs.hasAttribute(Attribute::InAlloca) &&
- !Attrs.hasAttribute(Attribute::Preallocated),
- "Attributes 'byval', 'inalloca', and 'preallocated' do not "
+ !Attrs.hasAttribute(Attribute::ByRef) &&
+ !Attrs.hasAttribute(Attribute::InAlloca) &&
+ !Attrs.hasAttribute(Attribute::Preallocated),
+ "Attributes 'byval', 'byref', 'inalloca', and 'preallocated' do not "
"support unsized types!",
V);
}
@@ -1731,10 +1793,28 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty,
"Attribute 'swifterror' only applies to parameters "
"with pointer to pointer type!",
V);
+
+ if (Attrs.hasAttribute(Attribute::ByRef)) {
+ Assert(Attrs.getByRefType() == PTy->getElementType(),
+ "Attribute 'byref' type does not match parameter!", V);
+ }
+
+ if (Attrs.hasAttribute(Attribute::ByVal) && Attrs.getByValType()) {
+ Assert(Attrs.getByValType() == PTy->getElementType(),
+ "Attribute 'byval' type does not match parameter!", V);
+ }
+
+ if (Attrs.hasAttribute(Attribute::Preallocated)) {
+ Assert(Attrs.getPreallocatedType() == PTy->getElementType(),
+ "Attribute 'preallocated' type does not match parameter!", V);
+ }
} else {
Assert(!Attrs.hasAttribute(Attribute::ByVal),
"Attribute 'byval' only applies to parameters with pointer type!",
V);
+ Assert(!Attrs.hasAttribute(Attribute::ByRef),
+ "Attribute 'byref' only applies to parameters with pointer type!",
+ V);
Assert(!Attrs.hasAttribute(Attribute::SwiftError),
"Attribute 'swifterror' only applies to parameters "
"with pointer type!",
@@ -1765,10 +1845,11 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
!RetAttrs.hasAttribute(Attribute::Returned) &&
!RetAttrs.hasAttribute(Attribute::InAlloca) &&
!RetAttrs.hasAttribute(Attribute::Preallocated) &&
+ !RetAttrs.hasAttribute(Attribute::ByRef) &&
!RetAttrs.hasAttribute(Attribute::SwiftSelf) &&
!RetAttrs.hasAttribute(Attribute::SwiftError)),
- "Attributes 'byval', 'inalloca', 'preallocated', 'nest', 'sret', "
- "'nocapture', 'nofree', "
+ "Attributes 'byval', 'inalloca', 'preallocated', 'byref', "
+ "'nest', 'sret', 'nocapture', 'nofree', "
"'returned', 'swiftself', and 'swifterror' do not apply to return "
"values!",
V);
@@ -2102,39 +2183,22 @@ void Verifier::verifyStatepoint(const CallBase &Call) {
Call);
const int NumTransitionArgs =
cast<ConstantInt>(NumTransitionArgsV)->getZExtValue();
- Assert(NumTransitionArgs >= 0,
- "gc.statepoint number of transition arguments must be positive", Call);
+ Assert(NumTransitionArgs == 0,
+ "gc.statepoint w/inline transition bundle is deprecated", Call);
const int EndTransitionArgsInx = EndCallArgsInx + 1 + NumTransitionArgs;
- // We're migrating away from inline operands to operand bundles, enforce
- // the either/or property during transition.
- if (Call.getOperandBundle(LLVMContext::OB_gc_transition)) {
- Assert(NumTransitionArgs == 0,
- "can't use both deopt operands and deopt bundle on a statepoint");
- }
-
const Value *NumDeoptArgsV = Call.getArgOperand(EndTransitionArgsInx + 1);
Assert(isa<ConstantInt>(NumDeoptArgsV),
"gc.statepoint number of deoptimization arguments "
"must be constant integer",
Call);
const int NumDeoptArgs = cast<ConstantInt>(NumDeoptArgsV)->getZExtValue();
- Assert(NumDeoptArgs >= 0,
- "gc.statepoint number of deoptimization arguments "
- "must be positive",
- Call);
+ Assert(NumDeoptArgs == 0,
+ "gc.statepoint w/inline deopt operands is deprecated", Call);
- // We're migrating away from inline operands to operand bundles, enforce
- // the either/or property during transition.
- if (Call.getOperandBundle(LLVMContext::OB_deopt)) {
- Assert(NumDeoptArgs == 0,
- "can't use both deopt operands and deopt bundle on a statepoint");
- }
-
- const int ExpectedNumArgs =
- 7 + NumCallArgs + NumTransitionArgs + NumDeoptArgs;
- Assert(ExpectedNumArgs <= (int)Call.arg_size(),
- "gc.statepoint too few arguments according to length fields", Call);
+ const int ExpectedNumArgs = 7 + NumCallArgs;
+ Assert(ExpectedNumArgs == (int)Call.arg_size(),
+ "gc.statepoint too many arguments", Call);
// Check that the only uses of this gc.statepoint are gc.result or
// gc.relocate calls which are tied to this statepoint and thus part
@@ -2280,6 +2344,11 @@ void Verifier::visitFunction(const Function &F) {
default:
case CallingConv::C:
break;
+ case CallingConv::X86_INTR: {
+ Assert(F.arg_empty() || Attrs.hasParamAttribute(0, Attribute::ByVal),
+ "Calling convention parameter requires byval", &F);
+ break;
+ }
case CallingConv::AMDGPU_KERNEL:
case CallingConv::SPIR_KERNEL:
Assert(F.getReturnType()->isVoidTy(),
@@ -2292,6 +2361,28 @@ void Verifier::visitFunction(const Function &F) {
case CallingConv::AMDGPU_CS:
Assert(!F.hasStructRetAttr(),
"Calling convention does not allow sret", &F);
+ if (F.getCallingConv() != CallingConv::SPIR_KERNEL) {
+ const unsigned StackAS = DL.getAllocaAddrSpace();
+ unsigned i = 0;
+ for (const Argument &Arg : F.args()) {
+ Assert(!Attrs.hasParamAttribute(i, Attribute::ByVal),
+ "Calling convention disallows byval", &F);
+ Assert(!Attrs.hasParamAttribute(i, Attribute::Preallocated),
+ "Calling convention disallows preallocated", &F);
+ Assert(!Attrs.hasParamAttribute(i, Attribute::InAlloca),
+ "Calling convention disallows inalloca", &F);
+
+ if (Attrs.hasParamAttribute(i, Attribute::ByRef)) {
+ // FIXME: Should also disallow LDS and GDS, but we don't have the enum
+ // value here.
+ Assert(Arg.getType()->getPointerAddressSpace() != StackAS,
+ "Calling convention disallows stack byref", &F);
+ }
+
+ ++i;
+ }
+ }
+
LLVM_FALLTHROUGH;
case CallingConv::Fast:
case CallingConv::Cold:
@@ -2394,6 +2485,10 @@ void Verifier::visitFunction(const Function &F) {
"function must have a single !dbg attachment", &F, I.second);
AssertDI(isa<DISubprogram>(I.second),
"function !dbg attachment must be a subprogram", &F, I.second);
+ AssertDI(cast<DISubprogram>(I.second)->isDistinct(),
+ "function definition may only have a distinct !dbg attachment",
+ &F);
+
auto *SP = cast<DISubprogram>(I.second);
const Function *&AttachedTo = DISubprogramAttachments[SP];
AssertDI(!AttachedTo || AttachedTo == &F,
@@ -2488,15 +2583,10 @@ void Verifier::visitBasicBlock(BasicBlock &BB) {
// Check constraints that this basic block imposes on all of the PHI nodes in
// it.
if (isa<PHINode>(BB.front())) {
- SmallVector<BasicBlock*, 8> Preds(pred_begin(&BB), pred_end(&BB));
+ SmallVector<BasicBlock *, 8> Preds(predecessors(&BB));
SmallVector<std::pair<BasicBlock*, Value*>, 8> Values;
llvm::sort(Preds);
for (const PHINode &PN : BB.phis()) {
- // Ensure that PHI nodes have at least one entry!
- Assert(PN.getNumIncomingValues() != 0,
- "PHI nodes must have at least one entry. If the block is dead, "
- "the PHI should be removed!",
- &PN);
Assert(PN.getNumIncomingValues() == Preds.size(),
"PHINode should have one entry for each predecessor of its "
"parent basic block!",
@@ -2887,8 +2977,8 @@ void Verifier::visitAddrSpaceCastInst(AddrSpaceCastInst &I) {
Assert(SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace(),
"AddrSpaceCast must be between different address spaces", &I);
if (auto *SrcVTy = dyn_cast<VectorType>(SrcTy))
- Assert(SrcVTy->getNumElements() ==
- cast<VectorType>(DestTy)->getNumElements(),
+ Assert(SrcVTy->getElementCount() ==
+ cast<VectorType>(DestTy)->getElementCount(),
"AddrSpaceCast vector pointer number of elements mismatch", &I);
visitInstruction(I);
}
@@ -3174,17 +3264,19 @@ static bool isTypeCongruent(Type *L, Type *R) {
static AttrBuilder getParameterABIAttributes(int I, AttributeList Attrs) {
static const Attribute::AttrKind ABIAttrs[] = {
- Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca,
- Attribute::InReg, Attribute::SwiftSelf, Attribute::SwiftError,
- Attribute::Preallocated};
+ Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca,
+ Attribute::InReg, Attribute::SwiftSelf, Attribute::SwiftError,
+ Attribute::Preallocated, Attribute::ByRef};
AttrBuilder Copy;
for (auto AK : ABIAttrs) {
if (Attrs.hasParamAttribute(I, AK))
Copy.addAttribute(AK);
}
- // `align` is ABI-affecting only in combination with `byval`.
+
+ // `align` is ABI-affecting only in combination with `byval` or `byref`.
if (Attrs.hasParamAttribute(I, Attribute::Alignment) &&
- Attrs.hasParamAttribute(I, Attribute::ByVal))
+ (Attrs.hasParamAttribute(I, Attribute::ByVal) ||
+ Attrs.hasParamAttribute(I, Attribute::ByRef)))
Copy.addAlignmentAttr(Attrs.getParamAlignment(I));
return Copy;
}
@@ -3420,7 +3512,7 @@ void Verifier::visitGetElementPtrInst(GetElementPtrInst &GEP) {
"GEP base pointer is not a vector or a vector of pointers", &GEP);
Assert(GEP.getSourceElementType()->isSized(), "GEP into unsized type!", &GEP);
- SmallVector<Value*, 16> Idxs(GEP.idx_begin(), GEP.idx_end());
+ SmallVector<Value *, 16> Idxs(GEP.indices());
Assert(all_of(
Idxs, [](Value* V) { return V->getType()->isIntOrIntVectorTy(); }),
"GEP indexes must be integers", &GEP);
@@ -4205,6 +4297,14 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) {
}
}
+void Verifier::visitAnnotationMetadata(MDNode *Annotation) {
+ Assert(isa<MDTuple>(Annotation), "annotation must be a tuple");
+ Assert(Annotation->getNumOperands() >= 1,
+ "annotation must have at least one operand");
+ for (const MDOperand &Op : Annotation->operands())
+ Assert(isa<MDString>(Op.get()), "operands must be strings");
+}
+
/// verifyInstruction - Verify that an instruction is well formed.
///
void Verifier::visitInstruction(Instruction &I) {
@@ -4274,7 +4374,7 @@ void Verifier::visitInstruction(Instruction &I) {
F->getIntrinsicID() == Intrinsic::experimental_patchpoint_void ||
F->getIntrinsicID() == Intrinsic::experimental_patchpoint_i64 ||
F->getIntrinsicID() == Intrinsic::experimental_gc_statepoint ||
- F->getIntrinsicID() == Intrinsic::wasm_rethrow_in_catch,
+ F->getIntrinsicID() == Intrinsic::wasm_rethrow,
"Cannot invoke an intrinsic other than donothing, patchpoint, "
"statepoint, coro_resume or coro_destroy",
&I);
@@ -4365,6 +4465,9 @@ void Verifier::visitInstruction(Instruction &I) {
if (MDNode *MD = I.getMetadata(LLVMContext::MD_prof))
visitProfMetadata(I, MD);
+ if (MDNode *Annotation = I.getMetadata(LLVMContext::MD_annotation))
+ visitAnnotationMetadata(Annotation);
+
if (MDNode *N = I.getDebugLoc().getAsMDNode()) {
AssertDI(isa<DILocation>(N), "invalid !dbg metadata attachment", &I, N);
visitMDNode(*N, AreDebugLocsAllowed::Yes);
@@ -4785,45 +4888,6 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
"gc.relocate: statepoint base index out of bounds", Call);
Assert(DerivedIndex < Opt->Inputs.size(),
"gc.relocate: statepoint derived index out of bounds", Call);
- } else {
- Assert(BaseIndex < StatepointCall.arg_size(),
- "gc.relocate: statepoint base index out of bounds", Call);
- Assert(DerivedIndex < StatepointCall.arg_size(),
- "gc.relocate: statepoint derived index out of bounds", Call);
-
- // Check that BaseIndex and DerivedIndex fall within the 'gc parameters'
- // section of the statepoint's argument.
- Assert(StatepointCall.arg_size() > 0,
- "gc.statepoint: insufficient arguments");
- Assert(isa<ConstantInt>(StatepointCall.getArgOperand(3)),
- "gc.statement: number of call arguments must be constant integer");
- const uint64_t NumCallArgs =
- cast<ConstantInt>(StatepointCall.getArgOperand(3))->getZExtValue();
- Assert(StatepointCall.arg_size() > NumCallArgs + 5,
- "gc.statepoint: mismatch in number of call arguments");
- Assert(isa<ConstantInt>(StatepointCall.getArgOperand(NumCallArgs + 5)),
- "gc.statepoint: number of transition arguments must be "
- "a constant integer");
- const uint64_t NumTransitionArgs =
- cast<ConstantInt>(StatepointCall.getArgOperand(NumCallArgs + 5))
- ->getZExtValue();
- const uint64_t DeoptArgsStart = 4 + NumCallArgs + 1 + NumTransitionArgs + 1;
- Assert(isa<ConstantInt>(StatepointCall.getArgOperand(DeoptArgsStart)),
- "gc.statepoint: number of deoptimization arguments must be "
- "a constant integer");
- const uint64_t NumDeoptArgs =
- cast<ConstantInt>(StatepointCall.getArgOperand(DeoptArgsStart))
- ->getZExtValue();
- const uint64_t GCParamArgsStart = DeoptArgsStart + 1 + NumDeoptArgs;
- const uint64_t GCParamArgsEnd = StatepointCall.arg_size();
- Assert(GCParamArgsStart <= BaseIndex && BaseIndex < GCParamArgsEnd,
- "gc.relocate: statepoint base index doesn't fall within the "
- "'gc parameters' section of the statepoint call",
- Call);
- Assert(GCParamArgsStart <= DerivedIndex && DerivedIndex < GCParamArgsEnd,
- "gc.relocate: statepoint derived index doesn't fall within the "
- "'gc parameters' section of the statepoint call",
- Call);
}
// Relocated value must be either a pointer type or vector-of-pointer type,
@@ -4952,15 +5016,17 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
case Intrinsic::sadd_sat:
case Intrinsic::uadd_sat:
case Intrinsic::ssub_sat:
- case Intrinsic::usub_sat: {
+ case Intrinsic::usub_sat:
+ case Intrinsic::sshl_sat:
+ case Intrinsic::ushl_sat: {
Value *Op1 = Call.getArgOperand(0);
Value *Op2 = Call.getArgOperand(1);
Assert(Op1->getType()->isIntOrIntVectorTy(),
- "first operand of [us][add|sub]_sat must be an int type or vector "
- "of ints");
+ "first operand of [us][add|sub|shl]_sat must be an int type or "
+ "vector of ints");
Assert(Op2->getType()->isIntOrIntVectorTy(),
- "second operand of [us][add|sub]_sat must be an int type or vector "
- "of ints");
+ "second operand of [us][add|sub|shl]_sat must be an int type or "
+ "vector of ints");
break;
}
case Intrinsic::smul_fix:
@@ -5013,6 +5079,14 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
Assert(Size % 16 == 0, "bswap must be an even number of bytes", &Call);
break;
}
+ case Intrinsic::invariant_start: {
+ ConstantInt *InvariantSize = dyn_cast<ConstantInt>(Call.getArgOperand(0));
+ Assert(InvariantSize &&
+ (!InvariantSize->isNegative() || InvariantSize->isMinusOne()),
+ "invariant_start parameter must be -1, 0 or a positive number",
+ &Call);
+ break;
+ }
case Intrinsic::matrix_multiply:
case Intrinsic::matrix_transpose:
case Intrinsic::matrix_column_major_load:
@@ -5076,7 +5150,7 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
"Vector element type mismatch of the result and second operand "
"vector!", IF);
- Assert(ResultTy->getNumElements() ==
+ Assert(cast<FixedVectorType>(ResultTy)->getNumElements() ==
NumRows->getZExtValue() * NumColumns->getZExtValue(),
"Result of a matrix operation does not fit in the returned vector!");
@@ -5086,6 +5160,30 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
break;
}
+ case Intrinsic::experimental_vector_insert: {
+ VectorType *VecTy = cast<VectorType>(Call.getArgOperand(0)->getType());
+ VectorType *SubVecTy = cast<VectorType>(Call.getArgOperand(1)->getType());
+
+ Assert(VecTy->getElementType() == SubVecTy->getElementType(),
+ "experimental_vector_insert parameters must have the same element "
+ "type.",
+ &Call);
+ break;
+ }
+ case Intrinsic::experimental_vector_extract: {
+ VectorType *ResultTy = cast<VectorType>(Call.getType());
+ VectorType *VecTy = cast<VectorType>(Call.getArgOperand(0)->getType());
+
+ Assert(ResultTy->getElementType() == VecTy->getElementType(),
+ "experimental_vector_extract result must have the same element "
+ "type as the input vector.",
+ &Call);
+ break;
+ }
+ case Intrinsic::experimental_noalias_scope_decl: {
+ NoAliasScopeDecls.push_back(cast<IntrinsicInst>(&Call));
+ break;
+ }
};
}
@@ -5162,7 +5260,7 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) {
Assert(Operand->getType()->isFPOrFPVectorTy(),
"Intrinsic first argument must be floating point", &FPI);
if (auto *OperandT = dyn_cast<VectorType>(Operand->getType())) {
- NumSrcElem = OperandT->getNumElements();
+ NumSrcElem = cast<FixedVectorType>(OperandT)->getNumElements();
}
Operand = &FPI;
@@ -5171,7 +5269,7 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) {
Assert(Operand->getType()->isIntOrIntVectorTy(),
"Intrinsic result must be an integer", &FPI);
if (auto *OperandT = dyn_cast<VectorType>(Operand->getType())) {
- Assert(NumSrcElem == OperandT->getNumElements(),
+ Assert(NumSrcElem == cast<FixedVectorType>(OperandT)->getNumElements(),
"Intrinsic first argument and result vector lengths must be equal",
&FPI);
}
@@ -5185,7 +5283,7 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) {
Assert(Operand->getType()->isIntOrIntVectorTy(),
"Intrinsic first argument must be integer", &FPI);
if (auto *OperandT = dyn_cast<VectorType>(Operand->getType())) {
- NumSrcElem = OperandT->getNumElements();
+ NumSrcElem = cast<FixedVectorType>(OperandT)->getNumElements();
}
Operand = &FPI;
@@ -5194,7 +5292,7 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) {
Assert(Operand->getType()->isFPOrFPVectorTy(),
"Intrinsic result must be a floating point", &FPI);
if (auto *OperandT = dyn_cast<VectorType>(Operand->getType())) {
- Assert(NumSrcElem == OperandT->getNumElements(),
+ Assert(NumSrcElem == cast<FixedVectorType>(OperandT)->getNumElements(),
"Intrinsic first argument and result vector lengths must be equal",
&FPI);
}
@@ -5213,9 +5311,8 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) {
Assert(OperandTy->isVectorTy() == ResultTy->isVectorTy(),
"Intrinsic first argument and result disagree on vector use", &FPI);
if (OperandTy->isVectorTy()) {
- auto *OperandVecTy = cast<VectorType>(OperandTy);
- auto *ResultVecTy = cast<VectorType>(ResultTy);
- Assert(OperandVecTy->getNumElements() == ResultVecTy->getNumElements(),
+ Assert(cast<FixedVectorType>(OperandTy)->getNumElements() ==
+ cast<FixedVectorType>(ResultTy)->getNumElements(),
"Intrinsic first argument and result vector lengths must be equal",
&FPI);
}
@@ -5437,6 +5534,75 @@ void Verifier::verifySourceDebugInfo(const DICompileUnit &U, const DIFile &F) {
"inconsistent use of embedded source");
}
+void Verifier::verifyNoAliasScopeDecl() {
+ if (NoAliasScopeDecls.empty())
+ return;
+
+ // only a single scope must be declared at a time.
+ for (auto *II : NoAliasScopeDecls) {
+ assert(II->getIntrinsicID() == Intrinsic::experimental_noalias_scope_decl &&
+ "Not a llvm.experimental.noalias.scope.decl ?");
+ const auto *ScopeListMV = dyn_cast<MetadataAsValue>(
+ II->getOperand(Intrinsic::NoAliasScopeDeclScopeArg));
+ Assert(ScopeListMV != nullptr,
+ "llvm.experimental.noalias.scope.decl must have a MetadataAsValue "
+ "argument",
+ II);
+
+ const auto *ScopeListMD = dyn_cast<MDNode>(ScopeListMV->getMetadata());
+ Assert(ScopeListMD != nullptr, "!id.scope.list must point to an MDNode",
+ II);
+ Assert(ScopeListMD->getNumOperands() == 1,
+ "!id.scope.list must point to a list with a single scope", II);
+ }
+
+ // Only check the domination rule when requested. Once all passes have been
+ // adapted this option can go away.
+ if (!VerifyNoAliasScopeDomination)
+ return;
+
+ // Now sort the intrinsics based on the scope MDNode so that declarations of
+ // the same scopes are next to each other.
+ auto GetScope = [](IntrinsicInst *II) {
+ const auto *ScopeListMV = cast<MetadataAsValue>(
+ II->getOperand(Intrinsic::NoAliasScopeDeclScopeArg));
+ return &cast<MDNode>(ScopeListMV->getMetadata())->getOperand(0);
+ };
+
+ // We are sorting on MDNode pointers here. For valid input IR this is ok.
+ // TODO: Sort on Metadata ID to avoid non-deterministic error messages.
+ auto Compare = [GetScope](IntrinsicInst *Lhs, IntrinsicInst *Rhs) {
+ return GetScope(Lhs) < GetScope(Rhs);
+ };
+
+ llvm::sort(NoAliasScopeDecls, Compare);
+
+ // Go over the intrinsics and check that for the same scope, they are not
+ // dominating each other.
+ auto ItCurrent = NoAliasScopeDecls.begin();
+ while (ItCurrent != NoAliasScopeDecls.end()) {
+ auto CurScope = GetScope(*ItCurrent);
+ auto ItNext = ItCurrent;
+ do {
+ ++ItNext;
+ } while (ItNext != NoAliasScopeDecls.end() &&
+ GetScope(*ItNext) == CurScope);
+
+ // [ItCurrent, ItNext) represents the declarations for the same scope.
+ // Ensure they are not dominating each other.. but only if it is not too
+ // expensive.
+ if (ItNext - ItCurrent < 32)
+ for (auto *I : llvm::make_range(ItCurrent, ItNext))
+ for (auto *J : llvm::make_range(ItCurrent, ItNext))
+ if (I != J)
+ Assert(!DT.dominates(I, J),
+ "llvm.experimental.noalias.scope.decl dominates another one "
+ "with the same scope",
+ I);
+ ItCurrent = ItNext;
+ }
+}
+
//===----------------------------------------------------------------------===//
// Implement the public interfaces to this file...
//===----------------------------------------------------------------------===//
generated by cgit v1.2.3 (git 2.25.1) at 2024-10-01 22:30:24 +0000