diff options
Diffstat (limited to 'llvm/lib/IR/Verifier.cpp')
| -rw-r--r-- | llvm/lib/IR/Verifier.cpp | 678 |
1 files changed, 518 insertions, 160 deletions
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index d15b70d71b47..6df1072925f9 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -397,6 +397,9 @@ public: } private: + /// Whether a metadata node is allowed to be, or contain, a DILocation. + enum class AreDebugLocsAllowed { No, Yes }; + // Verification methods... void visitGlobalValue(const GlobalValue &GV); void visitGlobalVariable(const GlobalVariable &GV); @@ -405,7 +408,7 @@ private: void visitAliaseeSubExpr(SmallPtrSetImpl<const GlobalAlias *> &Visited, const GlobalAlias &A, const Constant &C); void visitNamedMDNode(const NamedMDNode &NMD); - void visitMDNode(const MDNode &MD); + void visitMDNode(const MDNode &MD, AreDebugLocsAllowed AllowLocs); void visitMetadataAsValue(const MetadataAsValue &MD, Function *F); void visitValueAsMetadata(const ValueAsMetadata &MD, Function *F); void visitComdat(const Comdat &C); @@ -567,8 +570,9 @@ void Verifier::visitGlobalValue(const GlobalValue &GV) { Assert(!GV.isDeclaration() || GV.hasValidDeclarationLinkage(), "Global is external, but doesn't have external or weak linkage!", &GV); - Assert(GV.getAlignment() <= Value::MaximumAlignment, - "huge alignment values are unsupported", &GV); + if (const GlobalObject *GO = dyn_cast<GlobalObject>(&GV)) + Assert(GO->getAlignment() <= Value::MaximumAlignment, + "huge alignment values are unsupported", GO); Assert(!GV.hasAppendingLinkage() || isa<GlobalVariable>(GV), "Only global variables can have appending linkage!", &GV); @@ -590,15 +594,12 @@ void Verifier::visitGlobalValue(const GlobalValue &GV) { "Global is marked as dllimport, but not external", &GV); } - if (GV.hasLocalLinkage()) + if (GV.isImplicitDSOLocal()) Assert(GV.isDSOLocal(), - "GlobalValue with private or internal linkage must be dso_local!", + "GlobalValue with local linkage or non-default " + "visibility must be dso_local!", &GV); - if (!GV.hasDefaultVisibility() && !GV.hasExternalWeakLinkage()) - Assert(GV.isDSOLocal(), - "GlobalValue with non default visibility must be dso_local!", &GV); - forEachUser(&GV, GlobalValueVisited, [&](const Value *V) -> bool { if (const Instruction *I = dyn_cast<Instruction>(V)) { if (!I->getParent() || !I->getParent()->getParent()) @@ -701,8 +702,8 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) { // the runtime size. If the global is a struct or an array containing // scalable vectors, that will be caught by the isValidElementType methods // in StructType or ArrayType instead. - if (auto *VTy = dyn_cast<VectorType>(GV.getValueType())) - Assert(!VTy->isScalable(), "Globals cannot contain scalable vectors", &GV); + Assert(!isa<ScalableVectorType>(GV.getValueType()), + "Globals cannot contain scalable vectors", &GV); if (!GV.hasInitializer()) { visitGlobalValue(GV); @@ -783,11 +784,11 @@ void Verifier::visitNamedMDNode(const NamedMDNode &NMD) { if (!MD) continue; - visitMDNode(*MD); + visitMDNode(*MD, AreDebugLocsAllowed::Yes); } } -void Verifier::visitMDNode(const MDNode &MD) { +void Verifier::visitMDNode(const MDNode &MD, AreDebugLocsAllowed AllowLocs) { // Only visit each node once. Metadata can be mutually recursive, so this // avoids infinite recursion here, as well as being an optimization. if (!MDNodes.insert(&MD).second) @@ -810,8 +811,10 @@ void Verifier::visitMDNode(const MDNode &MD) { continue; Assert(!isa<LocalAsMetadata>(Op), "Invalid operand for global metadata!", &MD, Op); + AssertDI(!isa<DILocation>(Op) || AllowLocs == AreDebugLocsAllowed::Yes, + "DILocation not allowed within this metadata node", &MD, Op); if (auto *N = dyn_cast<MDNode>(Op)) { - visitMDNode(*N); + visitMDNode(*N, AllowLocs); continue; } if (auto *V = dyn_cast<ValueAsMetadata>(Op)) { @@ -854,7 +857,7 @@ void Verifier::visitValueAsMetadata(const ValueAsMetadata &MD, Function *F) { void Verifier::visitMetadataAsValue(const MetadataAsValue &MDV, Function *F) { Metadata *MD = MDV.getMetadata(); if (auto *N = dyn_cast<MDNode>(MD)) { - visitMDNode(*N); + visitMDNode(*N, AreDebugLocsAllowed::No); return; } @@ -891,12 +894,30 @@ void Verifier::visitDIScope(const DIScope &N) { void Verifier::visitDISubrange(const DISubrange &N) { AssertDI(N.getTag() == dwarf::DW_TAG_subrange_type, "invalid tag", &N); + AssertDI(N.getRawCountNode() || N.getRawUpperBound(), + "Subrange must contain count or upperBound", &N); + AssertDI(!N.getRawCountNode() || !N.getRawUpperBound(), + "Subrange can have any one of count or upperBound", &N); + AssertDI(!N.getRawCountNode() || N.getCount(), + "Count must either be a signed constant or a DIVariable", &N); auto Count = N.getCount(); - AssertDI(Count, "Count must either be a signed constant or a DIVariable", - &N); - AssertDI(!Count.is<ConstantInt*>() || - Count.get<ConstantInt*>()->getSExtValue() >= -1, + AssertDI(!Count || !Count.is<ConstantInt *>() || + Count.get<ConstantInt *>()->getSExtValue() >= -1, "invalid subrange count", &N); + auto *LBound = N.getRawLowerBound(); + AssertDI(!LBound || isa<ConstantAsMetadata>(LBound) || + isa<DIVariable>(LBound) || isa<DIExpression>(LBound), + "LowerBound must be signed constant or DIVariable or DIExpression", + &N); + auto *UBound = N.getRawUpperBound(); + AssertDI(!UBound || isa<ConstantAsMetadata>(UBound) || + isa<DIVariable>(UBound) || isa<DIExpression>(UBound), + "UpperBound must be signed constant or DIVariable or DIExpression", + &N); + auto *Stride = N.getRawStride(); + AssertDI(!Stride || isa<ConstantAsMetadata>(Stride) || + isa<DIVariable>(Stride) || isa<DIExpression>(Stride), + "Stride must be signed constant or DIVariable or DIExpression", &N); } void Verifier::visitDIEnumerator(const DIEnumerator &N) { @@ -1009,6 +1030,11 @@ void Verifier::visitDICompositeType(const DICompositeType &N) { AssertDI(isa<DIDerivedType>(D) && N.getTag() == dwarf::DW_TAG_variant_part, "discriminator can only appear on variant part"); } + + if (N.getRawDataLocation()) { + AssertDI(N.getTag() == dwarf::DW_TAG_array_type, + "dataLocation can only appear in array type"); + } } void Verifier::visitDISubroutineType(const DISubroutineType &N) { @@ -1037,6 +1063,9 @@ void Verifier::visitDIFile(const DIFile &N) { case DIFile::CSK_SHA1: Size = 40; break; + case DIFile::CSK_SHA256: + Size = 64; + break; } AssertDI(Checksum->Value.size() == Size, "invalid checksum length", &N); AssertDI(Checksum->Value.find_if_not(llvm::isHexDigit) == StringRef::npos, @@ -1250,7 +1279,9 @@ void Verifier::visitDIGlobalVariable(const DIGlobalVariable &N) { AssertDI(N.getTag() == dwarf::DW_TAG_variable, "invalid tag", &N); AssertDI(isType(N.getRawType()), "invalid type ref", &N, N.getRawType()); - AssertDI(N.getType(), "missing global variable type", &N); + // Assert only if the global variable is not an extern + if (N.isDefinition()) + AssertDI(N.getType(), "missing global variable type", &N); if (auto *Member = N.getRawStaticDataMemberDeclaration()) { AssertDI(isa<DIDerivedType>(Member), "invalid static data member declaration", &N, Member); @@ -1476,6 +1507,13 @@ Verifier::visitModuleFlag(const MDNode *Op, "'Linker Options' named metadata no longer supported"); } + if (ID->getString() == "SemanticInterposition") { + ConstantInt *Value = + mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(2)); + Assert(Value, + "SemanticInterposition metadata requires constant integer argument"); + } + if (ID->getString() == "CG Profile") { for (const MDOperand &MDO : cast<MDNode>(Op->getOperand(2))->operands()) visitModuleFlagCGProfileEntry(MDO); @@ -1502,6 +1540,7 @@ void Verifier::visitModuleFlagCGProfileEntry(const MDOperand &MDO) { /// Return true if this attribute kind only applies to functions. static bool isFuncOnlyAttr(Attribute::AttrKind Kind) { switch (Kind) { + case Attribute::NoMerge: case Attribute::NoReturn: case Attribute::NoSync: case Attribute::WillReturn: @@ -1545,6 +1584,7 @@ static bool isFuncOnlyAttr(Attribute::AttrKind Kind) { case Attribute::SpeculativeLoadHardening: case Attribute::Speculatable: case Attribute::StrictFP: + case Attribute::NullPointerIsValid: return true; default: break; @@ -1556,7 +1596,8 @@ static bool isFuncOnlyAttr(Attribute::AttrKind Kind) { /// arguments. static bool isFuncOrArgAttr(Attribute::AttrKind Kind) { return Kind == Attribute::ReadOnly || Kind == Attribute::WriteOnly || - Kind == Attribute::ReadNone || Kind == Attribute::NoFree; + Kind == Attribute::ReadNone || Kind == Attribute::NoFree || + Kind == Attribute::Preallocated; } void Verifier::verifyAttributeTypes(AttributeSet Attrs, bool IsFunction, @@ -1565,6 +1606,13 @@ void Verifier::verifyAttributeTypes(AttributeSet Attrs, bool IsFunction, if (A.isStringAttribute()) continue; + if (A.isIntAttribute() != + Attribute::doesAttrKindHaveArgument(A.getKindAsEnum())) { + CheckFailed("Attribute '" + A.getAsString() + "' should have an Argument", + V); + return; + } + if (isFuncOnlyAttr(A.getKindAsEnum())) { if (!IsFunction) { CheckFailed("Attribute '" + A.getAsString() + @@ -1600,11 +1648,13 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty, unsigned AttrCount = 0; AttrCount += Attrs.hasAttribute(Attribute::ByVal); AttrCount += Attrs.hasAttribute(Attribute::InAlloca); + AttrCount += Attrs.hasAttribute(Attribute::Preallocated); AttrCount += Attrs.hasAttribute(Attribute::StructRet) || Attrs.hasAttribute(Attribute::InReg); AttrCount += Attrs.hasAttribute(Attribute::Nest); - Assert(AttrCount <= 1, "Attributes 'byval', 'inalloca', 'inreg', 'nest', " - "and 'sret' are incompatible!", + Assert(AttrCount <= 1, + "Attributes 'byval', 'inalloca', 'preallocated', 'inreg', 'nest', " + "and 'sret' are incompatible!", V); Assert(!(Attrs.hasAttribute(Attribute::InAlloca) && @@ -1654,6 +1704,12 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty, "Attribute 'byval' type does not match parameter!", V); } + if (Attrs.hasAttribute(Attribute::Preallocated)) { + Assert(Attrs.getPreallocatedType() == + cast<PointerType>(Ty)->getElementType(), + "Attribute 'preallocated' type does not match parameter!", V); + } + AttrBuilder IncompatibleAttrs = AttributeFuncs::typeIncompatible(Ty); Assert(!AttrBuilder(Attrs).overlaps(IncompatibleAttrs), "Wrong types for attribute: " + @@ -1664,8 +1720,10 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty, SmallPtrSet<Type*, 4> Visited; if (!PTy->getElementType()->isSized(&Visited)) { Assert(!Attrs.hasAttribute(Attribute::ByVal) && - !Attrs.hasAttribute(Attribute::InAlloca), - "Attributes 'byval' and 'inalloca' do not support unsized types!", + !Attrs.hasAttribute(Attribute::InAlloca) && + !Attrs.hasAttribute(Attribute::Preallocated), + "Attributes 'byval', 'inalloca', and 'preallocated' do not " + "support unsized types!", V); } if (!isa<PointerType>(PTy->getElementType())) @@ -1706,9 +1764,11 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs, !RetAttrs.hasAttribute(Attribute::NoFree) && !RetAttrs.hasAttribute(Attribute::Returned) && !RetAttrs.hasAttribute(Attribute::InAlloca) && + !RetAttrs.hasAttribute(Attribute::Preallocated) && !RetAttrs.hasAttribute(Attribute::SwiftSelf) && !RetAttrs.hasAttribute(Attribute::SwiftError)), - "Attributes 'byval', 'inalloca', 'nest', 'sret', 'nocapture', 'nofree'" + "Attributes 'byval', 'inalloca', 'preallocated', 'nest', 'sret', " + "'nocapture', 'nofree', " "'returned', 'swiftself', and 'swifterror' do not apply to return " "values!", V); @@ -1852,16 +1912,25 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs, CheckFailed("invalid value for 'frame-pointer' attribute: " + FP, V); } + if (Attrs.hasFnAttribute("patchable-function-prefix")) { + StringRef S = Attrs + .getAttribute(AttributeList::FunctionIndex, + "patchable-function-prefix") + .getValueAsString(); + unsigned N; + if (S.getAsInteger(10, N)) + CheckFailed( + "\"patchable-function-prefix\" takes an unsigned integer: " + S, V); + } if (Attrs.hasFnAttribute("patchable-function-entry")) { - StringRef S0 = Attrs - .getAttribute(AttributeList::FunctionIndex, - "patchable-function-entry") - .getValueAsString(); - StringRef S = S0; + StringRef S = Attrs + .getAttribute(AttributeList::FunctionIndex, + "patchable-function-entry") + .getValueAsString(); unsigned N; if (S.getAsInteger(10, N)) CheckFailed( - "\"patchable-function-entry\" takes an unsigned integer: " + S0, V); + "\"patchable-function-entry\" takes an unsigned integer: " + S, V); } } @@ -2037,6 +2106,13 @@ void Verifier::verifyStatepoint(const CallBase &Call) { "gc.statepoint number of transition arguments must be positive", Call); const int EndTransitionArgsInx = EndCallArgsInx + 1 + NumTransitionArgs; + // We're migrating away from inline operands to operand bundles, enforce + // the either/or property during transition. + if (Call.getOperandBundle(LLVMContext::OB_gc_transition)) { + Assert(NumTransitionArgs == 0, + "can't use both deopt operands and deopt bundle on a statepoint"); + } + const Value *NumDeoptArgsV = Call.getArgOperand(EndTransitionArgsInx + 1); Assert(isa<ConstantInt>(NumDeoptArgsV), "gc.statepoint number of deoptimization arguments " @@ -2048,6 +2124,13 @@ void Verifier::verifyStatepoint(const CallBase &Call) { "must be positive", Call); + // We're migrating away from inline operands to operand bundles, enforce + // the either/or property during transition. + if (Call.getOperandBundle(LLVMContext::OB_deopt)) { + Assert(NumDeoptArgs == 0, + "can't use both deopt operands and deopt bundle on a statepoint"); + } + const int ExpectedNumArgs = 7 + NumCallArgs + NumTransitionArgs + NumDeoptArgs; Assert(ExpectedNumArgs <= (int)Call.arg_size(), @@ -2277,7 +2360,7 @@ void Verifier::visitFunction(const Function &F) { "function declaration may not have a !prof attachment", &F); // Verify the metadata itself. - visitMDNode(*I.second); + visitMDNode(*I.second, AreDebugLocsAllowed::Yes); } Assert(!F.hasPersonalityFn(), "Function declaration shouldn't have a personality routine", &F); @@ -2301,6 +2384,7 @@ void Verifier::visitFunction(const Function &F) { // Visit metadata attachments. for (const auto &I : MDs) { // Verify that the attachment is legal. + auto AllowLocs = AreDebugLocsAllowed::No; switch (I.first) { default: break; @@ -2315,6 +2399,7 @@ void Verifier::visitFunction(const Function &F) { AssertDI(!AttachedTo || AttachedTo == &F, "DISubprogram attached to more than one function", SP, &F); AttachedTo = &F; + AllowLocs = AreDebugLocsAllowed::Yes; break; } case LLVMContext::MD_prof: @@ -2325,7 +2410,7 @@ void Verifier::visitFunction(const Function &F) { } // Verify the metadata itself. - visitMDNode(*I.second); + visitMDNode(*I.second, AllowLocs); } } @@ -2344,8 +2429,7 @@ void Verifier::visitFunction(const Function &F) { if (!HasDebugInfo) return; - // Check that all !dbg attachments lead to back to N (or, at least, another - // subprogram that describes the same function). + // Check that all !dbg attachments lead to back to N. // // FIXME: Check this incrementally while visiting !dbg attachments. // FIXME: Only check when N is the canonical subprogram for F. @@ -2363,18 +2447,20 @@ void Verifier::visitFunction(const Function &F) { AssertDI(Parent && isa<DILocalScope>(Parent), "DILocation's scope must be a DILocalScope", N, &F, &I, DL, Parent); + DILocalScope *Scope = DL->getInlinedAtScope(); - if (Scope && !Seen.insert(Scope).second) + Assert(Scope, "Failed to find DILocalScope", DL); + + if (!Seen.insert(Scope).second) return; - DISubprogram *SP = Scope ? Scope->getSubprogram() : nullptr; + DISubprogram *SP = Scope->getSubprogram(); // Scope and SP could be the same MDNode and we don't want to skip // validation in that case if (SP && ((Scope != SP) && !Seen.insert(SP).second)) return; - // FIXME: Once N is canonical, check "SP == &N". AssertDI(SP->describes(&F), "!dbg attachment points at wrong subprogram for function", N, &F, &I, DL, Scope, SP); @@ -2513,8 +2599,6 @@ void Verifier::visitIndirectBrInst(IndirectBrInst &BI) { void Verifier::visitCallBrInst(CallBrInst &CBI) { Assert(CBI.isInlineAsm(), "Callbr is currently only used for asm-goto!", &CBI); - Assert(CBI.getType()->isVoidTy(), "Callbr return value is not supported!", - &CBI); for (unsigned i = 0, e = CBI.getNumSuccessors(); i != e; ++i) Assert(CBI.getSuccessor(i)->getType()->isLabelTy(), "Callbr successors must all have pointer type!", &CBI); @@ -2532,8 +2616,7 @@ void Verifier::visitCallBrInst(CallBrInst &CBI) { if (auto *BA = dyn_cast<BlockAddress>(V)) ArgBBs.insert(BA->getBasicBlock()); for (BasicBlock *BB : CBI.getIndirectDests()) - Assert(ArgBBs.find(BB) != ArgBBs.end(), - "Indirect label missing from arglist.", &CBI); + Assert(ArgBBs.count(BB), "Indirect label missing from arglist.", &CBI); } visitTerminator(CBI); @@ -2661,8 +2744,8 @@ void Verifier::visitUIToFPInst(UIToFPInst &I) { &I); if (SrcVec && DstVec) - Assert(cast<VectorType>(SrcTy)->getNumElements() == - cast<VectorType>(DestTy)->getNumElements(), + Assert(cast<VectorType>(SrcTy)->getElementCount() == + cast<VectorType>(DestTy)->getElementCount(), "UIToFP source and dest vector length mismatch", &I); visitInstruction(I); @@ -2684,8 +2767,8 @@ void Verifier::visitSIToFPInst(SIToFPInst &I) { &I); if (SrcVec && DstVec) - Assert(cast<VectorType>(SrcTy)->getNumElements() == - cast<VectorType>(DestTy)->getNumElements(), + Assert(cast<VectorType>(SrcTy)->getElementCount() == + cast<VectorType>(DestTy)->getElementCount(), "SIToFP source and dest vector length mismatch", &I); visitInstruction(I); @@ -2707,8 +2790,8 @@ void Verifier::visitFPToUIInst(FPToUIInst &I) { "FPToUI result must be integer or integer vector", &I); if (SrcVec && DstVec) - Assert(cast<VectorType>(SrcTy)->getNumElements() == - cast<VectorType>(DestTy)->getNumElements(), + Assert(cast<VectorType>(SrcTy)->getElementCount() == + cast<VectorType>(DestTy)->getElementCount(), "FPToUI source and dest vector length mismatch", &I); visitInstruction(I); @@ -2730,8 +2813,8 @@ void Verifier::visitFPToSIInst(FPToSIInst &I) { "FPToSI result must be integer or integer vector", &I); if (SrcVec && DstVec) - Assert(cast<VectorType>(SrcTy)->getNumElements() == - cast<VectorType>(DestTy)->getNumElements(), + Assert(cast<VectorType>(SrcTy)->getElementCount() == + cast<VectorType>(DestTy)->getElementCount(), "FPToSI source and dest vector length mismatch", &I); visitInstruction(I); @@ -2753,9 +2836,9 @@ void Verifier::visitPtrToIntInst(PtrToIntInst &I) { &I); if (SrcTy->isVectorTy()) { - VectorType *VSrc = cast<VectorType>(SrcTy); - VectorType *VDest = cast<VectorType>(DestTy); - Assert(VSrc->getNumElements() == VDest->getNumElements(), + auto *VSrc = cast<VectorType>(SrcTy); + auto *VDest = cast<VectorType>(DestTy); + Assert(VSrc->getElementCount() == VDest->getElementCount(), "PtrToInt Vector width mismatch", &I); } @@ -2778,9 +2861,9 @@ void Verifier::visitIntToPtrInst(IntToPtrInst &I) { Assert(SrcTy->isVectorTy() == DestTy->isVectorTy(), "IntToPtr type mismatch", &I); if (SrcTy->isVectorTy()) { - VectorType *VSrc = cast<VectorType>(SrcTy); - VectorType *VDest = cast<VectorType>(DestTy); - Assert(VSrc->getNumElements() == VDest->getNumElements(), + auto *VSrc = cast<VectorType>(SrcTy); + auto *VDest = cast<VectorType>(DestTy); + Assert(VSrc->getElementCount() == VDest->getElementCount(), "IntToPtr Vector width mismatch", &I); } visitInstruction(I); @@ -2803,8 +2886,9 @@ void Verifier::visitAddrSpaceCastInst(AddrSpaceCastInst &I) { &I); Assert(SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace(), "AddrSpaceCast must be between different address spaces", &I); - if (SrcTy->isVectorTy()) - Assert(SrcTy->getVectorNumElements() == DestTy->getVectorNumElements(), + if (auto *SrcVTy = dyn_cast<VectorType>(SrcTy)) + Assert(SrcVTy->getNumElements() == + cast<VectorType>(DestTy)->getNumElements(), "AddrSpaceCast vector pointer number of elements mismatch", &I); visitInstruction(I); } @@ -2836,9 +2920,9 @@ void Verifier::visitPHINode(PHINode &PN) { } void Verifier::visitCallBase(CallBase &Call) { - Assert(Call.getCalledValue()->getType()->isPointerTy(), + Assert(Call.getCalledOperand()->getType()->isPointerTy(), "Called function must be a pointer!", Call); - PointerType *FPTy = cast<PointerType>(Call.getCalledValue()->getType()); + PointerType *FPTy = cast<PointerType>(Call.getCalledOperand()->getType()); Assert(FPTy->getElementType()->isFunctionTy(), "Called function is not pointer to function type!", Call); @@ -2871,16 +2955,23 @@ void Verifier::visitCallBase(CallBase &Call) { bool IsIntrinsic = Call.getCalledFunction() && Call.getCalledFunction()->getName().startswith("llvm."); - Function *Callee - = dyn_cast<Function>(Call.getCalledValue()->stripPointerCasts()); + Function *Callee = + dyn_cast<Function>(Call.getCalledOperand()->stripPointerCasts()); - if (Attrs.hasAttribute(AttributeList::FunctionIndex, Attribute::Speculatable)) { + if (Attrs.hasFnAttribute(Attribute::Speculatable)) { // Don't allow speculatable on call sites, unless the underlying function // declaration is also speculatable. Assert(Callee && Callee->isSpeculatable(), "speculatable attribute may not apply to call sites", Call); } + if (Attrs.hasFnAttribute(Attribute::Preallocated)) { + Assert(Call.getCalledFunction()->getIntrinsicID() == + Intrinsic::call_preallocated_arg, + "preallocated as a call site attribute can only be on " + "llvm.call.preallocated.arg"); + } + // Verify call attributes. verifyFunctionAttrs(FTy, Attrs, &Call, IsIntrinsic); @@ -2927,6 +3018,17 @@ void Verifier::visitCallBase(CallBase &Call) { Assert(isa<ConstantInt>(ArgVal) || isa<ConstantFP>(ArgVal), "immarg operand has non-immediate parameter", ArgVal, Call); } + + if (Call.paramHasAttr(i, Attribute::Preallocated)) { + Value *ArgVal = Call.getArgOperand(i); + bool hasOB = + Call.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0; + bool isMustTail = Call.isMustTailCall(); + Assert(hasOB != isMustTail, + "preallocated operand either requires a preallocated bundle or " + "the call to be musttail (but not both)", + ArgVal, Call); + } } if (FTy->isVarArg()) { @@ -2997,9 +3099,11 @@ void Verifier::visitCallBase(CallBase &Call) { visitIntrinsicCall(ID, Call); // Verify that a callsite has at most one "deopt", at most one "funclet", at - // most one "gc-transition", and at most one "cfguardtarget" operand bundle. + // most one "gc-transition", at most one "cfguardtarget", + // and at most one "preallocated" operand bundle. bool FoundDeoptBundle = false, FoundFuncletBundle = false, - FoundGCTransitionBundle = false, FoundCFGuardTargetBundle = false; + FoundGCTransitionBundle = false, FoundCFGuardTargetBundle = false, + FoundPreallocatedBundle = false, FoundGCLiveBundle = false;; for (unsigned i = 0, e = Call.getNumOperandBundles(); i < e; ++i) { OperandBundleUse BU = Call.getOperandBundleAt(i); uint32_t Tag = BU.getTagID(); @@ -3024,6 +3128,22 @@ void Verifier::visitCallBase(CallBase &Call) { FoundCFGuardTargetBundle = true; Assert(BU.Inputs.size() == 1, "Expected exactly one cfguardtarget bundle operand", Call); + } else if (Tag == LLVMContext::OB_preallocated) { + Assert(!FoundPreallocatedBundle, "Multiple preallocated operand bundles", + Call); + FoundPreallocatedBundle = true; + Assert(BU.Inputs.size() == 1, + "Expected exactly one preallocated bundle operand", Call); + auto Input = dyn_cast<IntrinsicInst>(BU.Inputs.front()); + Assert(Input && + Input->getIntrinsicID() == Intrinsic::call_preallocated_setup, + "\"preallocated\" argument must be a token from " + "llvm.call.preallocated.setup", + Call); + } else if (Tag == LLVMContext::OB_gc_live) { + Assert(!FoundGCLiveBundle, "Multiple gc-live operand bundles", + Call); + FoundGCLiveBundle = true; } } @@ -3054,15 +3174,17 @@ static bool isTypeCongruent(Type *L, Type *R) { static AttrBuilder getParameterABIAttributes(int I, AttributeList Attrs) { static const Attribute::AttrKind ABIAttrs[] = { - Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca, - Attribute::InReg, Attribute::Returned, Attribute::SwiftSelf, - Attribute::SwiftError}; + Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca, + Attribute::InReg, Attribute::SwiftSelf, Attribute::SwiftError, + Attribute::Preallocated}; AttrBuilder Copy; for (auto AK : ABIAttrs) { if (Attrs.hasParamAttribute(I, AK)) Copy.addAttribute(AK); } - if (Attrs.hasParamAttribute(I, Attribute::Alignment)) + // `align` is ABI-affecting only in combination with `byval`. + if (Attrs.hasParamAttribute(I, Attribute::Alignment) && + Attrs.hasParamAttribute(I, Attribute::ByVal)) Copy.addAlignmentAttr(Attrs.getParamAlignment(I)); return Copy; } @@ -3096,7 +3218,7 @@ void Verifier::verifyMustTailCall(CallInst &CI) { "cannot guarantee tail call due to mismatched calling conv", &CI); // - All ABI-impacting function attributes, such as sret, byval, inreg, - // returned, and inalloca, must match. + // returned, preallocated, and inalloca, must match. AttributeList CallerAttrs = F->getAttributes(); AttributeList CalleeAttrs = CI.getAttributes(); for (int I = 0, E = CallerTy->getNumParams(); I != E; ++I) { @@ -3154,7 +3276,7 @@ void Verifier::visitInvokeInst(InvokeInst &II) { /// visitUnaryOperator - Check the argument to the unary operator. /// void Verifier::visitUnaryOperator(UnaryOperator &U) { - Assert(U.getType() == U.getOperand(0)->getType(), + Assert(U.getType() == U.getOperand(0)->getType(), "Unary operators must have same type for" "operands and result!", &U); @@ -3286,7 +3408,7 @@ void Verifier::visitInsertElementInst(InsertElementInst &IE) { void Verifier::visitShuffleVectorInst(ShuffleVectorInst &SV) { Assert(ShuffleVectorInst::isValidOperands(SV.getOperand(0), SV.getOperand(1), - SV.getOperand(2)), + SV.getShuffleMask()), "Invalid shufflevector operands!", &SV); visitInstruction(SV); } @@ -3310,16 +3432,18 @@ void Verifier::visitGetElementPtrInst(GetElementPtrInst &GEP) { GEP.getResultElementType() == ElTy, "GEP is not of right type for indices!", &GEP, ElTy); - if (GEP.getType()->isVectorTy()) { + if (auto *GEPVTy = dyn_cast<VectorType>(GEP.getType())) { // Additional checks for vector GEPs. - unsigned GEPWidth = GEP.getType()->getVectorNumElements(); + ElementCount GEPWidth = GEPVTy->getElementCount(); if (GEP.getPointerOperandType()->isVectorTy()) - Assert(GEPWidth == GEP.getPointerOperandType()->getVectorNumElements(), - "Vector GEP result width doesn't match operand's", &GEP); + Assert( + GEPWidth == + cast<VectorType>(GEP.getPointerOperandType())->getElementCount(), + "Vector GEP result width doesn't match operand's", &GEP); for (Value *Idx : Idxs) { Type *IndexTy = Idx->getType(); - if (IndexTy->isVectorTy()) { - unsigned IndexWidth = IndexTy->getVectorNumElements(); + if (auto *IndexVTy = dyn_cast<VectorType>(IndexTy)) { + ElementCount IndexWidth = IndexVTy->getElementCount(); Assert(IndexWidth == GEPWidth, "Invalid GEP index vector width", &GEP); } Assert(IndexTy->isIntOrIntVectorTy(), @@ -4050,23 +4174,28 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) { // Check consistency of !prof branch_weights metadata. if (ProfName.equals("branch_weights")) { - unsigned ExpectedNumOperands = 0; - if (BranchInst *BI = dyn_cast<BranchInst>(&I)) - ExpectedNumOperands = BI->getNumSuccessors(); - else if (SwitchInst *SI = dyn_cast<SwitchInst>(&I)) - ExpectedNumOperands = SI->getNumSuccessors(); - else if (isa<CallInst>(&I) || isa<InvokeInst>(&I)) - ExpectedNumOperands = 1; - else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(&I)) - ExpectedNumOperands = IBI->getNumDestinations(); - else if (isa<SelectInst>(&I)) - ExpectedNumOperands = 2; - else - CheckFailed("!prof branch_weights are not allowed for this instruction", - MD); + if (isa<InvokeInst>(&I)) { + Assert(MD->getNumOperands() == 2 || MD->getNumOperands() == 3, + "Wrong number of InvokeInst branch_weights operands", MD); + } else { + unsigned ExpectedNumOperands = 0; + if (BranchInst *BI = dyn_cast<BranchInst>(&I)) + ExpectedNumOperands = BI->getNumSuccessors(); + else if (SwitchInst *SI = dyn_cast<SwitchInst>(&I)) + ExpectedNumOperands = SI->getNumSuccessors(); + else if (isa<CallInst>(&I)) + ExpectedNumOperands = 1; + else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(&I)) + ExpectedNumOperands = IBI->getNumDestinations(); + else if (isa<SelectInst>(&I)) + ExpectedNumOperands = 2; + else + CheckFailed("!prof branch_weights are not allowed for this instruction", + MD); - Assert(MD->getNumOperands() == 1 + ExpectedNumOperands, - "Wrong number of operands", MD); + Assert(MD->getNumOperands() == 1 + ExpectedNumOperands, + "Wrong number of operands", MD); + } for (unsigned i = 1; i < MD->getNumOperands(); ++i) { auto &MDO = MD->getOperand(i); Assert(MDO, "second operand should not be null", MD); @@ -4238,7 +4367,7 @@ void Verifier::visitInstruction(Instruction &I) { if (MDNode *N = I.getDebugLoc().getAsMDNode()) { AssertDI(isa<DILocation>(N), "invalid !dbg metadata attachment", &I, N); - visitMDNode(*N); + visitMDNode(*N, AreDebugLocsAllowed::Yes); } if (auto *DII = dyn_cast<DbgVariableIntrinsic>(&I)) { @@ -4246,6 +4375,17 @@ void Verifier::visitInstruction(Instruction &I) { verifyNotEntryValue(*DII); } + SmallVector<std::pair<unsigned, MDNode *>, 4> MDs; + I.getAllMetadata(MDs); + for (auto Attachment : MDs) { + unsigned Kind = Attachment.first; + auto AllowLocs = + (Kind == LLVMContext::MD_dbg || Kind == LLVMContext::MD_loop) + ? AreDebugLocsAllowed::Yes + : AreDebugLocsAllowed::No; + visitMDNode(*Attachment.second, AllowLocs); + } + InstsInThisBlock.insert(&I); } @@ -4304,6 +4444,41 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { switch (ID) { default: break; + case Intrinsic::assume: { + for (auto &Elem : Call.bundle_op_infos()) { + Assert(Elem.Tag->getKey() == "ignore" || + Attribute::isExistingAttribute(Elem.Tag->getKey()), + "tags must be valid attribute names"); + Attribute::AttrKind Kind = + Attribute::getAttrKindFromName(Elem.Tag->getKey()); + unsigned ArgCount = Elem.End - Elem.Begin; + if (Kind == Attribute::Alignment) { + Assert(ArgCount <= 3 && ArgCount >= 2, + "alignment assumptions should have 2 or 3 arguments"); + Assert(Call.getOperand(Elem.Begin)->getType()->isPointerTy(), + "first argument should be a pointer"); + Assert(Call.getOperand(Elem.Begin + 1)->getType()->isIntegerTy(), + "second argument should be an integer"); + if (ArgCount == 3) + Assert(Call.getOperand(Elem.Begin + 2)->getType()->isIntegerTy(), + "third argument should be an integer if present"); + return; + } + Assert(ArgCount <= 2, "to many arguments"); + if (Kind == Attribute::None) + break; + if (Attribute::doesAttrKindHaveArgument(Kind)) { + Assert(ArgCount == 2, "this attribute should have 2 arguments"); + Assert(isa<ConstantInt>(Call.getOperand(Elem.Begin + 1)), + "the second argument should be a constant integral value"); + } else if (isFuncOnlyAttr(Kind)) { + Assert((ArgCount) == 0, "this attribute has no argument"); + } else if (!isFuncOrArgAttr(Kind)) { + Assert((ArgCount) == 1, "this attribute should have one argument"); + } + } + break; + } case Intrinsic::coro_id: { auto *InfoArg = Call.getArgOperand(3)->stripPointerCasts(); if (isa<ConstantPointerNull>(InfoArg)) @@ -4318,7 +4493,7 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { "an array"); break; } -#define INSTRUCTION(NAME, NARGS, ROUND_MODE, INTRINSIC, DAGN) \ +#define INSTRUCTION(NAME, NARGS, ROUND_MODE, INTRINSIC) \ case Intrinsic::INTRINSIC: #include "llvm/IR/ConstrainedOps.def" visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(Call)); @@ -4338,6 +4513,7 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { visitDbgLabelIntrinsic("label", cast<DbgLabelInst>(Call)); break; case Intrinsic::memcpy: + case Intrinsic::memcpy_inline: case Intrinsic::memmove: case Intrinsic::memset: { const auto *MI = cast<MemIntrinsic>(&Call); @@ -4368,15 +4544,6 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { "must be a power of 2", Call); - if (auto *LengthCI = dyn_cast<ConstantInt>(AMI->getLength())) { - uint64_t Length = LengthCI->getZExtValue(); - uint64_t ElementSize = AMI->getElementSizeInBytes(); - Assert((Length % ElementSize) == 0, - "constant length must be a multiple of the element size in the " - "element-wise atomic memory intrinsic", - Call); - } - auto IsValidAlignment = [&](uint64_t Alignment) { return isPowerOf2_64(Alignment) && ElementSizeVal.ule(Alignment); }; @@ -4390,6 +4557,85 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { } break; } + case Intrinsic::call_preallocated_setup: { + auto *NumArgs = dyn_cast<ConstantInt>(Call.getArgOperand(0)); + Assert(NumArgs != nullptr, + "llvm.call.preallocated.setup argument must be a constant"); + bool FoundCall = false; + for (User *U : Call.users()) { + auto *UseCall = dyn_cast<CallBase>(U); + Assert(UseCall != nullptr, + "Uses of llvm.call.preallocated.setup must be calls"); + const Function *Fn = UseCall->getCalledFunction(); + if (Fn && Fn->getIntrinsicID() == Intrinsic::call_preallocated_arg) { + auto *AllocArgIndex = dyn_cast<ConstantInt>(UseCall->getArgOperand(1)); + Assert(AllocArgIndex != nullptr, + "llvm.call.preallocated.alloc arg index must be a constant"); + auto AllocArgIndexInt = AllocArgIndex->getValue(); + Assert(AllocArgIndexInt.sge(0) && + AllocArgIndexInt.slt(NumArgs->getValue()), + "llvm.call.preallocated.alloc arg index must be between 0 and " + "corresponding " + "llvm.call.preallocated.setup's argument count"); + } else if (Fn && Fn->getIntrinsicID() == + Intrinsic::call_preallocated_teardown) { + // nothing to do + } else { + Assert(!FoundCall, "Can have at most one call corresponding to a " + "llvm.call.preallocated.setup"); + FoundCall = true; + size_t NumPreallocatedArgs = 0; + for (unsigned i = 0; i < UseCall->getNumArgOperands(); i++) { + if (UseCall->paramHasAttr(i, Attribute::Preallocated)) { + ++NumPreallocatedArgs; + } + } + Assert(NumPreallocatedArgs != 0, + "cannot use preallocated intrinsics on a call without " + "preallocated arguments"); + Assert(NumArgs->equalsInt(NumPreallocatedArgs), + "llvm.call.preallocated.setup arg size must be equal to number " + "of preallocated arguments " + "at call site", + Call, *UseCall); + // getOperandBundle() cannot be called if more than one of the operand + // bundle exists. There is already a check elsewhere for this, so skip + // here if we see more than one. + if (UseCall->countOperandBundlesOfType(LLVMContext::OB_preallocated) > + 1) { + return; + } + auto PreallocatedBundle = + UseCall->getOperandBundle(LLVMContext::OB_preallocated); + Assert(PreallocatedBundle, + "Use of llvm.call.preallocated.setup outside intrinsics " + "must be in \"preallocated\" operand bundle"); + Assert(PreallocatedBundle->Inputs.front().get() == &Call, + "preallocated bundle must have token from corresponding " + "llvm.call.preallocated.setup"); + } + } + break; + } + case Intrinsic::call_preallocated_arg: { + auto *Token = dyn_cast<CallBase>(Call.getArgOperand(0)); + Assert(Token && Token->getCalledFunction()->getIntrinsicID() == + Intrinsic::call_preallocated_setup, + "llvm.call.preallocated.arg token argument must be a " + "llvm.call.preallocated.setup"); + Assert(Call.hasFnAttr(Attribute::Preallocated), + "llvm.call.preallocated.arg must be called with a \"preallocated\" " + "call site attribute"); + break; + } + case Intrinsic::call_preallocated_teardown: { + auto *Token = dyn_cast<CallBase>(Call.getArgOperand(0)); + Assert(Token && Token->getCalledFunction()->getIntrinsicID() == + Intrinsic::call_preallocated_setup, + "llvm.call.preallocated.teardown token argument must be a " + "llvm.call.preallocated.setup"); + break; + } case Intrinsic::gcroot: case Intrinsic::gcwrite: case Intrinsic::gcread: @@ -4506,20 +4752,20 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { LandingPad->getParent()); Assert(InvokeBB->getTerminator(), "safepoint block should be well formed", InvokeBB); - Assert(isStatepoint(InvokeBB->getTerminator()), + Assert(isa<GCStatepointInst>(InvokeBB->getTerminator()), "gc relocate should be linked to a statepoint", InvokeBB); } else { // In all other cases relocate should be tied to the statepoint directly. // This covers relocates on a normal return path of invoke statepoint and // relocates of a call statepoint. auto Token = Call.getArgOperand(0); - Assert(isa<Instruction>(Token) && isStatepoint(cast<Instruction>(Token)), + Assert(isa<GCStatepointInst>(Token), "gc relocate is incorrectly tied to the statepoint", Call, Token); } // Verify rest of the relocate arguments. const CallBase &StatepointCall = - *cast<CallBase>(cast<GCRelocateInst>(Call).getStatepoint()); + *cast<GCRelocateInst>(Call).getStatepoint(); // Both the base and derived must be piped through the safepoint. Value *Base = Call.getArgOperand(1); @@ -4530,47 +4776,55 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { Assert(isa<ConstantInt>(Derived), "gc.relocate operand #3 must be integer offset", Call); - const int BaseIndex = cast<ConstantInt>(Base)->getZExtValue(); - const int DerivedIndex = cast<ConstantInt>(Derived)->getZExtValue(); + const uint64_t BaseIndex = cast<ConstantInt>(Base)->getZExtValue(); + const uint64_t DerivedIndex = cast<ConstantInt>(Derived)->getZExtValue(); + // Check the bounds - Assert(0 <= BaseIndex && BaseIndex < (int)StatepointCall.arg_size(), - "gc.relocate: statepoint base index out of bounds", Call); - Assert(0 <= DerivedIndex && DerivedIndex < (int)StatepointCall.arg_size(), - "gc.relocate: statepoint derived index out of bounds", Call); - - // Check that BaseIndex and DerivedIndex fall within the 'gc parameters' - // section of the statepoint's argument. - Assert(StatepointCall.arg_size() > 0, - "gc.statepoint: insufficient arguments"); - Assert(isa<ConstantInt>(StatepointCall.getArgOperand(3)), - "gc.statement: number of call arguments must be constant integer"); - const unsigned NumCallArgs = + if (auto Opt = StatepointCall.getOperandBundle(LLVMContext::OB_gc_live)) { + Assert(BaseIndex < Opt->Inputs.size(), + "gc.relocate: statepoint base index out of bounds", Call); + Assert(DerivedIndex < Opt->Inputs.size(), + "gc.relocate: statepoint derived index out of bounds", Call); + } else { + Assert(BaseIndex < StatepointCall.arg_size(), + "gc.relocate: statepoint base index out of bounds", Call); + Assert(DerivedIndex < StatepointCall.arg_size(), + "gc.relocate: statepoint derived index out of bounds", Call); + + // Check that BaseIndex and DerivedIndex fall within the 'gc parameters' + // section of the statepoint's argument. + Assert(StatepointCall.arg_size() > 0, + "gc.statepoint: insufficient arguments"); + Assert(isa<ConstantInt>(StatepointCall.getArgOperand(3)), + "gc.statement: number of call arguments must be constant integer"); + const uint64_t NumCallArgs = cast<ConstantInt>(StatepointCall.getArgOperand(3))->getZExtValue(); - Assert(StatepointCall.arg_size() > NumCallArgs + 5, - "gc.statepoint: mismatch in number of call arguments"); - Assert(isa<ConstantInt>(StatepointCall.getArgOperand(NumCallArgs + 5)), - "gc.statepoint: number of transition arguments must be " - "a constant integer"); - const int NumTransitionArgs = - cast<ConstantInt>(StatepointCall.getArgOperand(NumCallArgs + 5)) - ->getZExtValue(); - const int DeoptArgsStart = 4 + NumCallArgs + 1 + NumTransitionArgs + 1; - Assert(isa<ConstantInt>(StatepointCall.getArgOperand(DeoptArgsStart)), - "gc.statepoint: number of deoptimization arguments must be " - "a constant integer"); - const int NumDeoptArgs = - cast<ConstantInt>(StatepointCall.getArgOperand(DeoptArgsStart)) - ->getZExtValue(); - const int GCParamArgsStart = DeoptArgsStart + 1 + NumDeoptArgs; - const int GCParamArgsEnd = StatepointCall.arg_size(); - Assert(GCParamArgsStart <= BaseIndex && BaseIndex < GCParamArgsEnd, - "gc.relocate: statepoint base index doesn't fall within the " - "'gc parameters' section of the statepoint call", - Call); - Assert(GCParamArgsStart <= DerivedIndex && DerivedIndex < GCParamArgsEnd, - "gc.relocate: statepoint derived index doesn't fall within the " - "'gc parameters' section of the statepoint call", - Call); + Assert(StatepointCall.arg_size() > NumCallArgs + 5, + "gc.statepoint: mismatch in number of call arguments"); + Assert(isa<ConstantInt>(StatepointCall.getArgOperand(NumCallArgs + 5)), + "gc.statepoint: number of transition arguments must be " + "a constant integer"); + const uint64_t NumTransitionArgs = + cast<ConstantInt>(StatepointCall.getArgOperand(NumCallArgs + 5)) + ->getZExtValue(); + const uint64_t DeoptArgsStart = 4 + NumCallArgs + 1 + NumTransitionArgs + 1; + Assert(isa<ConstantInt>(StatepointCall.getArgOperand(DeoptArgsStart)), + "gc.statepoint: number of deoptimization arguments must be " + "a constant integer"); + const uint64_t NumDeoptArgs = + cast<ConstantInt>(StatepointCall.getArgOperand(DeoptArgsStart)) + ->getZExtValue(); + const uint64_t GCParamArgsStart = DeoptArgsStart + 1 + NumDeoptArgs; + const uint64_t GCParamArgsEnd = StatepointCall.arg_size(); + Assert(GCParamArgsStart <= BaseIndex && BaseIndex < GCParamArgsEnd, + "gc.relocate: statepoint base index doesn't fall within the " + "'gc parameters' section of the statepoint call", + Call); + Assert(GCParamArgsStart <= DerivedIndex && DerivedIndex < GCParamArgsEnd, + "gc.relocate: statepoint derived index doesn't fall within the " + "'gc parameters' section of the statepoint call", + Call); + } // Relocated value must be either a pointer type or vector-of-pointer type, // but gc_relocate does not need to return the same pointer type as the @@ -4598,6 +4852,14 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { "eh.exceptionpointer argument must be a catchpad", Call); break; } + case Intrinsic::get_active_lane_mask: { + Assert(Call.getType()->isVectorTy(), "get_active_lane_mask: must return a " + "vector", Call); + auto *ElemTy = Call.getType()->getScalarType(); + Assert(ElemTy->isIntegerTy(1), "get_active_lane_mask: element type is not " + "i1", Call); + break; + } case Intrinsic::masked_load: { Assert(Call.getType()->isVectorTy(), "masked_load: must return a vector", Call); @@ -4617,8 +4879,8 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { "masked_load: return must match pointer type", Call); Assert(PassThru->getType() == DataTy, "masked_load: pass through and data type must match", Call); - Assert(Mask->getType()->getVectorNumElements() == - DataTy->getVectorNumElements(), + Assert(cast<VectorType>(Mask->getType())->getElementCount() == + cast<VectorType>(DataTy)->getElementCount(), "masked_load: vector mask must be same length as data", Call); break; } @@ -4636,12 +4898,27 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { Type *DataTy = cast<PointerType>(Ptr->getType())->getElementType(); Assert(DataTy == Val->getType(), "masked_store: storee must match pointer type", Call); - Assert(Mask->getType()->getVectorNumElements() == - DataTy->getVectorNumElements(), + Assert(cast<VectorType>(Mask->getType())->getElementCount() == + cast<VectorType>(DataTy)->getElementCount(), "masked_store: vector mask must be same length as data", Call); break; } + case Intrinsic::masked_gather: { + const APInt &Alignment = + cast<ConstantInt>(Call.getArgOperand(1))->getValue(); + Assert(Alignment.isNullValue() || Alignment.isPowerOf2(), + "masked_gather: alignment must be 0 or a power of 2", Call); + break; + } + case Intrinsic::masked_scatter: { + const APInt &Alignment = + cast<ConstantInt>(Call.getArgOperand(2))->getValue(); + Assert(Alignment.isNullValue() || Alignment.isPowerOf2(), + "masked_scatter: alignment must be 0 or a power of 2", Call); + break; + } + case Intrinsic::experimental_guard: { Assert(isa<CallInst>(Call), "experimental_guard cannot be invoked", Call); Assert(Call.countOperandBundlesOfType(LLVMContext::OB_deopt) == 1, @@ -4691,7 +4968,9 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { case Intrinsic::umul_fix: case Intrinsic::umul_fix_sat: case Intrinsic::sdiv_fix: - case Intrinsic::udiv_fix: { + case Intrinsic::sdiv_fix_sat: + case Intrinsic::udiv_fix: + case Intrinsic::udiv_fix_sat: { Value *Op1 = Call.getArgOperand(0); Value *Op2 = Call.getArgOperand(1); Assert(Op1->getType()->isIntOrIntVectorTy(), @@ -4706,7 +4985,7 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { "third argument of [us][mul|div]_fix[_sat] must fit within 32 bits"); if (ID == Intrinsic::smul_fix || ID == Intrinsic::smul_fix_sat || - ID == Intrinsic::sdiv_fix) { + ID == Intrinsic::sdiv_fix || ID == Intrinsic::sdiv_fix_sat) { Assert( Op3->getZExtValue() < Op1->getType()->getScalarSizeInBits(), "the scale of s[mul|div]_fix[_sat] must be less than the width of " @@ -4728,6 +5007,85 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { "Intrinsic does not support vectors", &Call); break; } + case Intrinsic::bswap: { + Type *Ty = Call.getType(); + unsigned Size = Ty->getScalarSizeInBits(); + Assert(Size % 16 == 0, "bswap must be an even number of bytes", &Call); + break; + } + case Intrinsic::matrix_multiply: + case Intrinsic::matrix_transpose: + case Intrinsic::matrix_column_major_load: + case Intrinsic::matrix_column_major_store: { + Function *IF = Call.getCalledFunction(); + ConstantInt *Stride = nullptr; + ConstantInt *NumRows; + ConstantInt *NumColumns; + VectorType *ResultTy; + Type *Op0ElemTy = nullptr; + Type *Op1ElemTy = nullptr; + switch (ID) { + case Intrinsic::matrix_multiply: + NumRows = cast<ConstantInt>(Call.getArgOperand(2)); + NumColumns = cast<ConstantInt>(Call.getArgOperand(4)); + ResultTy = cast<VectorType>(Call.getType()); + Op0ElemTy = + cast<VectorType>(Call.getArgOperand(0)->getType())->getElementType(); + Op1ElemTy = + cast<VectorType>(Call.getArgOperand(1)->getType())->getElementType(); + break; + case Intrinsic::matrix_transpose: + NumRows = cast<ConstantInt>(Call.getArgOperand(1)); + NumColumns = cast<ConstantInt>(Call.getArgOperand(2)); + ResultTy = cast<VectorType>(Call.getType()); + Op0ElemTy = + cast<VectorType>(Call.getArgOperand(0)->getType())->getElementType(); + break; + case Intrinsic::matrix_column_major_load: + Stride = dyn_cast<ConstantInt>(Call.getArgOperand(1)); + NumRows = cast<ConstantInt>(Call.getArgOperand(3)); + NumColumns = cast<ConstantInt>(Call.getArgOperand(4)); + ResultTy = cast<VectorType>(Call.getType()); + Op0ElemTy = + cast<PointerType>(Call.getArgOperand(0)->getType())->getElementType(); + break; + case Intrinsic::matrix_column_major_store: + Stride = dyn_cast<ConstantInt>(Call.getArgOperand(2)); + NumRows = cast<ConstantInt>(Call.getArgOperand(4)); + NumColumns = cast<ConstantInt>(Call.getArgOperand(5)); + ResultTy = cast<VectorType>(Call.getArgOperand(0)->getType()); + Op0ElemTy = + cast<VectorType>(Call.getArgOperand(0)->getType())->getElementType(); + Op1ElemTy = + cast<PointerType>(Call.getArgOperand(1)->getType())->getElementType(); + break; + default: + llvm_unreachable("unexpected intrinsic"); + } + + Assert(ResultTy->getElementType()->isIntegerTy() || + ResultTy->getElementType()->isFloatingPointTy(), + "Result type must be an integer or floating-point type!", IF); + + Assert(ResultTy->getElementType() == Op0ElemTy, + "Vector element type mismatch of the result and first operand " + "vector!", IF); + + if (Op1ElemTy) + Assert(ResultTy->getElementType() == Op1ElemTy, + "Vector element type mismatch of the result and second operand " + "vector!", IF); + + Assert(ResultTy->getNumElements() == + NumRows->getZExtValue() * NumColumns->getZExtValue(), + "Result of a matrix operation does not fit in the returned vector!"); + + if (Stride) + Assert(Stride->getZExtValue() >= NumRows->getZExtValue(), + "Stride must be greater or equal than the number of rows!", IF); + + break; + } }; } @@ -4754,7 +5112,7 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) { unsigned NumOperands; bool HasRoundingMD; switch (FPI.getIntrinsicID()) { -#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ +#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ case Intrinsic::INTRINSIC: \ NumOperands = NARG; \ HasRoundingMD = ROUND_MODE; \ @@ -4777,7 +5135,7 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) { Type *ResultTy = FPI.getType(); Assert(!ValTy->isVectorTy() && !ResultTy->isVectorTy(), "Intrinsic does not support vectors", &FPI); - } + } break; case Intrinsic::experimental_constrained_lround: @@ -4787,7 +5145,7 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) { Assert(!ValTy->isVectorTy() && !ResultTy->isVectorTy(), "Intrinsic does not support vectors", &FPI); break; - } + } case Intrinsic::experimental_constrained_fcmp: case Intrinsic::experimental_constrained_fcmps: { @@ -4798,7 +5156,7 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) { } case Intrinsic::experimental_constrained_fptosi: - case Intrinsic::experimental_constrained_fptoui: { + case Intrinsic::experimental_constrained_fptoui: { Value *Operand = FPI.getArgOperand(0); uint64_t NumSrcElem = 0; Assert(Operand->getType()->isFPOrFPVectorTy(), @@ -4870,7 +5228,7 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) { "Intrinsic first argument's type must be smaller than result type", &FPI); } - } + } break; default: @@ -5136,7 +5494,7 @@ struct VerifierLegacyPass : public FunctionPass { bool runOnFunction(Function &F) override { if (!V->verify(F) && FatalErrors) { - errs() << "in function " << F.getName() << '\n'; + errs() << "in function " << F.getName() << '\n'; report_fatal_error("Broken function found, compilation aborted!"); } return false; |